【2023年】Python PlaywrightでWebスクレイピング(3) – URLリストによるデータ収集とall_inner_texts()を用いた一括取得

#====================================================
# main.py
# (url1.csvのデータ収集)
# 
# V1.0 2023.4.23  : 初版
#====================================================
import pandas as pd
import time
import myscraping

start = time.time() 

# csv 読み込み
df_csv = pd.read_csv("url1.csv",index_col=0)

# scrapying report 生成
print(f'No,Result,Category,URL')
for i in range(len(df_csv)):
    category = df_csv.iloc[i]['Category']
    url      = df_csv.iloc[i]['URL']

    # スクレイピング実行
    result, result_df = myscraping.scrape(url)
    print(f'{i},{result}, {category}, {url}')

    # 結果出力
    result_csv_name = "output/"+category+".csv"
    result_df.to_csv(result_csv_name, index=False)
     
end = time.time()
elapsed_time = end - start
print(f'経過時間:{elapsed_time:.2f} 秒')

from playwright.sync_api import Playwright, sync_playwright, expect
import pandas as pd

def playwright_run(playwright: Playwright, url, df):
    
    browser = playwright.chromium.launch(headless=True)
    context = browser.new_context()

    #### 1.Open url page
    page = context.new_page()
    page.goto(url)
    
    #### 2.情報取得
    # Title取得
    titles = page.locator('h2.entry-card-title.card-title.e-card-title').all_inner_texts()
    # print(titles)
    
    # 登録日取得
    entry_dates = page.locator('span.entry-date').all_inner_texts()
    # print(entry_dates)
        
    # Close page
    page.close()
    context.close()
    browser.close()
    
    #### 3.収集結果データ作成 
    new_df = pd.DataFrame({'Title': titles,'Entry_date': entry_dates})
    df = pd.concat([df, new_df], ignore_index=True)
    # print(df)
    return "OK", df


def scrape(url):
    result = "OK"

    # 結果出力の空フレーム作成
    cols = ['Title','Entry_date']
    df = pd.DataFrame(columns=cols)

    # Playwright 実行
    with sync_playwright() as playwright:
        result,df = playwright_run(playwright,url,df)
    
    return result, df