async def scrape(url: str) -> str: from playwright.async_api import async_playwright async with async_playwright() as p: browser = await p.chromium.launch(headless=True) context = await browser.new_context(viewport={"width": 1000, "height": 2000}) page = await context.new_page() await page.goto(url, timeout=60000) # Wait until at least one toggle button is present await page.wait_for_selector(".toggle-btn", timeout=20000) # Set zoom await page.evaluate("document.body.style.zoom='50%'") # Find all toggle buttons toggle_buttons = await page.query_selector_all(".toggle-btn") print(f"Found {len(toggle_buttons)} toggle buttons") for i, btn in enumerate(toggle_buttons): try: # Ensure it's visible and enabled if await btn.is_visible() and await btn.is_enabled(): await btn.click() await page.wait_for_timeout(1000) if i == len(toggle_buttons) - 1: break # Scroll down gradually scroll_step = 500 total_height = await page.evaluate("() => document.body.scrollHeight") current_position = 0 while current_position < total_height: await page.evaluate(f"window.scrollTo(0, {current_position});") await page.wait_for_timeout(100) current_position += scroll_step total_height = await page.evaluate( "() => document.body.scrollHeight" ) except Exception as e: print(f"Skipped button due to error: {e}") # Get the page content page_source = await page.content() # Close the browser await browser.close() # Continue scraping logic here... print("Scraping done") # Save the page content to a file with open("/cache/scraped_page.html", "w") as fp: fp.write(page_source) return page_source