rik/dagster
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor

Browse Source
This commit is contained in:
Rik Veenboer
2025-08-16 13:56:18 +02:00
parent 4593b97bc2
commit 4242638818
3 changed files with 7 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
apps/vinyl/src/assets.py
View File

@@ -13,10 +13,10 @@ from jinja2 import Environment, FileSystemLoader
from models import Deal from models import Deal
from partitions import daily_partitions_def, multi_partitions_def from partitions import daily_partitions_def, multi_partitions_def
from plato.parse import parse as parse_plato from plato.parse import parse as parse_plato
from plato.scrape import scrape_plato from plato.scrape import scrape as scrape_plato
from shared.utils import get_partition_keys, parse_partition_keys from shared.utils import get_partition_keys, parse_partition_keys
from sounds.parse import parse as parse_sounds from sounds.parse import parse as parse_sounds
from sounds.scrape import scrape_sounds from sounds.scrape import scrape as scrape_sounds
from utils.email import EmailService from utils.email import EmailService
import dagster as dg import dagster as dg

2
apps/vinyl/src/plato/scrape.py
View File

@@ -6,7 +6,7 @@ import requests
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
def scrape_plato(get=None): def scrape(get=None):
ic() ic()
url = "https://www.platomania.nl/vinyl-aanbiedingen?page=1" url = "https://www.platomania.nl/vinyl-aanbiedingen?page=1"

8
apps/vinyl/src/sounds/scrape.py
View File

@@ -71,11 +71,11 @@ def parse_page(html_content):
) )
def scrape_sounds(): def scrape():
# Get page count # Get page count
page_count = get_page_count( response = requests.get("https://www.sounds.nl/uitverkoop/1/lp/all/art")
requests.get("https://www.sounds.nl/uitverkoop/1/lp/all/art").text response.raise_for_status()
) page_count = get_page_count(response.text)
time.sleep(1) time.sleep(1)
print(f"Number of pages: {page_count}") print(f"Number of pages: {page_count}")