move utils

apps/stocks/src/utils/scraper.py

@@ -0,0 +1,60 @@
+async def scrape(url: str) -> str:
+    from playwright.async_api import async_playwright
+
+    async with async_playwright() as p:
+        browser = await p.chromium.launch(headless=True)
+        context = await browser.new_context(viewport={"width": 1000, "height": 2000})
+        page = await context.new_page()
+
+        await page.goto(url, timeout=60000)
+
+        # Wait until at least one toggle button is present
+        await page.wait_for_selector(".toggle-btn", timeout=20000)
+
+        # Set zoom
+        await page.evaluate("document.body.style.zoom='50%'")
+
+        # Find all toggle buttons
+        toggle_buttons = await page.query_selector_all(".toggle-btn")
+        print(f"Found {len(toggle_buttons)} toggle buttons")
+
+        for i, btn in enumerate(toggle_buttons):
+            try:
+                # Ensure it's visible and enabled
+                if await btn.is_visible() and await btn.is_enabled():
+                    await btn.click()
+                    await page.wait_for_timeout(1000)
+
+                if i == len(toggle_buttons) - 1:
+                    break
+
+                # Scroll down gradually
+                scroll_step = 500
+                total_height = await page.evaluate("() => document.body.scrollHeight")
+                current_position = 0
+
+                while current_position < total_height:
+                    await page.evaluate(f"window.scrollTo(0, {current_position});")
+                    await page.wait_for_timeout(100)
+                    current_position += scroll_step
+                    total_height = await page.evaluate(
+                        "() => document.body.scrollHeight"
+                    )
+
+            except Exception as e:
+                print(f"Skipped button due to error: {e}")
+
+        # Get the page content
+        page_source = await page.content()
+
+        # Close the browser
+        await browser.close()
+
+        # Continue scraping logic here...
+        print("Scraping done")
+
+        # Save the page content to a file
+        with open("/cache/scraped_page.html", "w") as fp:
+            fp.write(page_source)
+
+    return page_source