정적 크롤링

명언 크롤링

 

import requests as rq
from bs4 import BeautifulSoup
import time

text_list = []
author_list = []
infor_list = []

for i in range (1,100):
  url = f'https://quotes.toscrape.com/page/{i}/'
  quote = rq.get(url)
  quote_html = BeautifulSoup(quote.content, 'html.parser')

  quote_text = quote_html.select('div.quote > span.text')
  quote_text_list = [i.text for i in quote_text]

  quote_author = quote_html.select('div.quote > span > small.author')
  quote_author_list = [i.text for i in quote_author]

  quote_link = quote_html.select('div.quote > span > a')
  quote_link_list = ['https://quotes.toscrape.com'+ i['href'] for i in quote_link]

  if len(quote_text_list) > 0:
    text_list.extend(quote_text_list)
    author_list.extend(quote_author_list)
    infor_list.extend(quote_link_list)
    time.sleep(1)
  else:
    break

import pandas as pd
df = pd.DataFrame({'text':text_list, 'author':author_list, 'infor':infor_list})
print(df.head())

결과

 

네이버 금융 속보 크롤링

 

import requests as rq
from bs4 import BeautifulSoup

url = 'https://finance.naver.com/news/news_list.naver?mode=LSS2D&section_id=101&section_id2=258'
data = rq.get(url)
html = BeautifulSoup(data.content, 'html.parser')
html_select = html.select('dl > dd.articleSubject > a')

title_list = [i['title'] for i in html_select]
print(title_list)

결과

 

한국거래소 상장공시시스템 (오늘의 공시)

 

#POST 방식
import requests as rq
from bs4 import BeautifulSoup
import pandas as pd

url = 'https://kind.krx.co.kr/disclosure/todaydisclosure.do'
payload = {
    'method': 'searchTodayDisclosureSub',
    'currentPageSize': '15',
    'pageIndex': '1',
    'orderMode': '0',
    'orderStat': 'D',
    'forward':'todaydisclosure_sub',
    'chose': 'S',
    'todayFlag': 'N',
    'selDate':'2026-04-30'
}

data = rq.post(url, data=payload)
html = BeautifulSoup(data.content, 'html.parser')

html_unicode =html.prettify()
tbl = pd.read_html(html.prettify())
print(tbl[0].head())

결과

 

 

※해당 예시는 도서 <퀀트 투자 포트폴리오 만들기>를 참고하여 작성하였습니다.