diff --git a/apps/vinyl/src/assets.py b/apps/vinyl/src/assets.py index 4754f8f..5bdc893 100644 --- a/apps/vinyl/src/assets.py +++ b/apps/vinyl/src/assets.py @@ -13,10 +13,10 @@ from jinja2 import Environment, FileSystemLoader from models import Deal from partitions import daily_partitions_def, multi_partitions_def from plato.parse import parse as parse_plato -from plato.scrape import scrape_plato +from plato.scrape import scrape as scrape_plato from shared.utils import get_partition_keys, parse_partition_keys from sounds.parse import parse as parse_sounds -from sounds.scrape import scrape_sounds +from sounds.scrape import scrape as scrape_sounds from utils.email import EmailService import dagster as dg diff --git a/apps/vinyl/src/plato/scrape.py b/apps/vinyl/src/plato/scrape.py index d05c2cb..b49f8af 100755 --- a/apps/vinyl/src/plato/scrape.py +++ b/apps/vinyl/src/plato/scrape.py @@ -6,7 +6,7 @@ import requests from bs4 import BeautifulSoup -def scrape_plato(get=None): +def scrape(get=None): ic() url = "https://www.platomania.nl/vinyl-aanbiedingen?page=1" diff --git a/apps/vinyl/src/sounds/scrape.py b/apps/vinyl/src/sounds/scrape.py index fd98333..bebe88f 100644 --- a/apps/vinyl/src/sounds/scrape.py +++ b/apps/vinyl/src/sounds/scrape.py @@ -71,11 +71,11 @@ def parse_page(html_content): ) -def scrape_sounds(): +def scrape(): # Get page count - page_count = get_page_count( - requests.get("https://www.sounds.nl/uitverkoop/1/lp/all/art").text - ) + response = requests.get("https://www.sounds.nl/uitverkoop/1/lp/all/art") + response.raise_for_status() + page_count = get_page_count(response.text) time.sleep(1) print(f"Number of pages: {page_count}")