61 lines
2.1 KiB
Python
61 lines
2.1 KiB
Python
async def scrape(url: str) -> str:
|
|
from playwright.async_api import async_playwright
|
|
|
|
async with async_playwright() as p:
|
|
browser = await p.chromium.launch(headless=True)
|
|
context = await browser.new_context(viewport={"width": 1000, "height": 2000})
|
|
page = await context.new_page()
|
|
|
|
await page.goto(url, timeout=60000)
|
|
|
|
# Wait until at least one toggle button is present
|
|
await page.wait_for_selector(".toggle-btn", timeout=20000)
|
|
|
|
# Set zoom
|
|
await page.evaluate("document.body.style.zoom='50%'")
|
|
|
|
# Find all toggle buttons
|
|
toggle_buttons = await page.query_selector_all(".toggle-btn")
|
|
print(f"Found {len(toggle_buttons)} toggle buttons")
|
|
|
|
for i, btn in enumerate(toggle_buttons):
|
|
try:
|
|
# Ensure it's visible and enabled
|
|
if await btn.is_visible() and await btn.is_enabled():
|
|
await btn.click()
|
|
await page.wait_for_timeout(1000)
|
|
|
|
if i == len(toggle_buttons) - 1:
|
|
break
|
|
|
|
# Scroll down gradually
|
|
scroll_step = 500
|
|
total_height = await page.evaluate("() => document.body.scrollHeight")
|
|
current_position = 0
|
|
|
|
while current_position < total_height:
|
|
await page.evaluate(f"window.scrollTo(0, {current_position});")
|
|
await page.wait_for_timeout(100)
|
|
current_position += scroll_step
|
|
total_height = await page.evaluate(
|
|
"() => document.body.scrollHeight"
|
|
)
|
|
|
|
except Exception as e:
|
|
print(f"Skipped button due to error: {e}")
|
|
|
|
# Get the page content
|
|
page_source = await page.content()
|
|
|
|
# Close the browser
|
|
await browser.close()
|
|
|
|
# Continue scraping logic here...
|
|
print("Scraping done")
|
|
|
|
# Save the page content to a file
|
|
with open("/cache/scraped_page.html", "w") as fp:
|
|
fp.write(page_source)
|
|
|
|
return page_source
|