Detect Cloudflare block pages and pace the Best Matches feed

README.md

@@ -158,6 +158,33 @@ check to complete before submitting each step, so no manual action is normally r
 browser profile usually keeps the Cloudflare clearance cookie and skips the visible check on later
 runs.
 
+### Cloudflare block pages
+
+Occasionally Cloudflare serves its **"Sorry, you have been blocked"** / **"Attention Required!"** page
+instead of Upwork. That page is a request refusal, not a login failure: the request never reaches
+Upwork, so no cookie, reload, or wait inside the browser will get past it. It is Cloudflare's bot
+scoring rejecting the automated browser session. The same machine and network usually still load
+Upwork normally in Chrome, and a `curl` request from that machine is blocked outright, so it does not
+indicate a network fault.
+
+The scraper detects that page and stops immediately with a message naming the block instead of
+scrolling an empty feed. Every run also logs the HTTP outcome of each Upwork document load,
+including the status, `cf-ray`, `cf-mitigated`, and `retry-after` values, for example:
+
+```
+[INFO] Response 403 for https://www.upwork.com/nx/find-work/best-matches (cf-ray=9d2f...-LHR, cf-mitigated=challenge)
+[INFO] Response 200 for https://www.upwork.com/nx/s/find-work/best-matches (cf-ray=9d2f...-LHR, cf-mitigated=-)
+```
+
+Because each retry from the same session extends the block:
+
+* Do not re-run the scraper in a loop after a block. Wait several minutes and try once.
+* Confirm the network is fine by opening Upwork in your normal browser.
+* Reusing an aged browser profile helps: log in once manually in the scraper's visible browser and
+  browse a little, so the profile carries normal cookies before automated runs reuse it.
+* A proxy (`UPWORK_PROXY_SERVER`) only helps when the route itself is the problem. If your normal
+  browser works from the same connection, changing proxies will not raise the automation score.
+
 The browser major version is detected automatically. Users should not normally maintain a Chrome version list.
 
 If Upwork reports network restrictions, configure the same stable proxy route that works in

tests/test_app.py

@@ -1,15 +1,19 @@
+import json
+import logging
 import sqlite3
 from pathlib import Path
 from types import SimpleNamespace
 
 from upwork_scraper import app
+from upwork_scraper import auth as app_auth
 from upwork_scraper.browser import BrowserSpec
 from upwork_scraper.config import Settings
 from upwork_scraper.database import create_db
 
 
-class FakeLogger:
+class FakeLogger(logging.Logger):
     def __init__(self):
+        super().__init__("fake")
         self.messages = []
 
     def info(self, *args, **kwargs):
@@ -18,6 +22,9 @@ def info(self, *args, **kwargs):
     def warning(self, *args, **kwargs):
         self.messages.append(("warning", args))
 
+    def error(self, *args, **kwargs):
+        self.messages.append(("error", args))
+
     def exception(self, *args, **kwargs):
         self.messages.append(("exception", args))
 
@@ -29,6 +36,12 @@ def __init__(self):
     def quit(self):
         self.closed = True
 
+    def execute_cdp_cmd(self, _command, _params):
+        return {}
+
+    def get_log(self, _log_type):
+        return []
+
 
 def test_validate_login_does_not_open_database(monkeypatch):
     settings = Settings(username="user@example.com", password="secret", first_name="FirstName")
@@ -67,14 +80,17 @@ def test_cli_converts_failure_to_one(monkeypatch):
 def test_job_urls_are_built_from_feed_tiles():
     class Driver:
         def execute_script(self, _script):
-            return [
-                {
-                    "uid": "2105718172637605907",
-                    "title": "PromptOps Engineer for Deterministic AI Agents",
-                },
-                {"uid": None, "title": "Missing opening UID"},
-                {"uid": "2105364230841134304", "title": "Build HIPAA-Compliant AI system"},
-            ]
+            return {
+                "tiles": [
+                    {
+                        "uid": "2105718172637605907",
+                        "title": "PromptOps Engineer for Deterministic AI Agents",
+                    },
+                    {"uid": None, "title": "Missing opening UID"},
+                    {"uid": "2105364230841134304", "title": "Build HIPAA-Compliant AI system"},
+                ],
+                "links": [],
+            }
 
     assert app._job_urls(Driver()) == [
         "https://www.upwork.com/jobs/PromptOps-Engineer-for-Deterministic-AI-Agents"
@@ -83,6 +99,23 @@ def execute_script(self, _script):
     ]
 
 
+def test_job_urls_fall_back_to_feed_links_when_tiles_have_no_hooks():
+    class Driver:
+        def execute_script(self, _script):
+            return {
+                "tiles": [{"href": None, "uid": None, "title": None}],
+                "links": [
+                    "/jobs/Ops-Engineer_~022116093407321073630/?referrer_url_path=find_work_home",
+                    "/jobs/Ops-Engineer_~022116093407321073630/?referrer_url_path=find_work_home",
+                    "/nx/search/jobs/?ontology_skill_uid=1031626712264720384",
+                ],
+            }
+
+    assert app._job_urls(Driver()) == [
+        "https://www.upwork.com/jobs/Ops-Engineer_~022116093407321073630/"
+    ]
+
+
 def test_load_job_list_waits_for_stable_empty_results(monkeypatch):
     class Driver:
         def __init__(self):
@@ -112,6 +145,7 @@ def until(self, condition):
     driver = Driver()
     logger = FakeLogger()
     monkeypatch.setattr(app, "SCROLL_STEPS", 2)
+    monkeypatch.setattr(app, "FEED_READY_TIMEOUT", 0)
     monkeypatch.setattr(app.time, "sleep", lambda _seconds: None)
     monkeypatch.setattr(app, "WebDriverWait", ImmediateWait)
 
@@ -121,6 +155,162 @@ def until(self, condition):
     assert any("Job list loaded" in message[0] for _, message in logger.messages)
 
 
+def test_wait_for_feed_returns_as_soon_as_tiles_render(monkeypatch):
+    class Driver:
+        def __init__(self):
+            self.tile_probes = 0
+
+        def execute_script(self, script):
+            if "job-tile" in script:
+                self.tile_probes += 1
+                return 0 if self.tile_probes < 2 else 30
+            return False
+
+    driver = Driver()
+    logger = FakeLogger()
+    monkeypatch.setattr(app.time, "sleep", lambda _seconds: None)
+
+    app._wait_for_feed(driver, logger)
+
+    assert driver.tile_probes == 2
+    assert not any(level == "warning" for level, _ in logger.messages)
+
+
+def test_job_urls_prefer_the_tile_link_and_strip_tracking_parameters():
+    class Driver:
+        def execute_script(self, _script):
+            return {
+                "tiles": [
+                    {
+                        "href": "/jobs/ML-Ops-Engineer_~022116093407321073630/"
+                        "?referrer_url_path=find_work_home",
+                        "uid": "2116093407321073630",
+                        "title": "ML Ops Engineer for Cloud Features",
+                    },
+                    {
+                        "href": "https://www.upwork.com/jobs/Absolute_~02200?x=1",
+                        "uid": None,
+                        "title": None,
+                    },
+                    {"href": None, "uid": None, "title": "Unusable tile"},
+                ],
+                "links": ["/jobs/ignored_~02999"],
+            }
+
+    assert app._job_urls(Driver()) == [
+        "https://www.upwork.com/jobs/ML-Ops-Engineer_~022116093407321073630/",
+        "https://www.upwork.com/jobs/Absolute_~02200",
+    ]
+
+
+def test_load_job_list_stops_on_cloudflare_block(monkeypatch):
+    """A block page must end the run instead of scrolling an empty page."""
+
+    class Driver:
+        current_url = "https://www.upwork.com/nx/find-work/best-matches"
+        scrolls = 0
+
+        def execute_script(self, script, *arguments):
+            if script == app_auth.BLOCKED_PAGE_JS:
+                return True
+            self.scrolls += 1
+            return None
+
+    driver = Driver()
+    try:
+        app._load_job_list(driver, FakeLogger())
+    except app_auth.UpworkBlockedError:
+        pass
+    else:
+        raise AssertionError("expected UpworkBlockedError before scrolling")
+    assert driver.scrolls == 0
+
+
+def test_document_responses_extract_cloudflare_verdicts():
+    class Driver:
+        def get_log(self, _log_type):
+            return [
+                {
+                    "message": json.dumps(
+                        {
+                            "message": {
+                                "method": "Network.responseReceived",
+                                "params": {
+                                    "type": "Document",
+                                    "response": {
+                                        "url": "https://www.upwork.com/nx/s/find-work/best-matches",
+                                        "status": 403,
+                                        "headers": {
+                                            "CF-Ray": "a5296037484ded07-LHR",
+                                            "cf-mitigated": "challenge",
+                                            "Retry-After": "60",
+                                        },
+                                    },
+                                },
+                            }
+                        }
+                    )
+                },
+                {
+                    "message": json.dumps(
+                        {
+                            "message": {
+                                "method": "Network.responseReceived",
+                                "params": {
+                                    "type": "XHR",
+                                    "response": {"url": "https://api", "status": 200},
+                                },
+                            }
+                        }
+                    )
+                },
+                {"message": "not json"},
+            ]
+
+    documents = app._document_responses(Driver())
+
+    assert len(documents) == 1
+    assert documents[0]["status"] == 403
+    assert documents[0]["cf-ray"] == "a5296037484ded07-LHR"
+    assert documents[0]["cf-mitigated"] == "challenge"
+    assert documents[0]["retry-after"] == "60"
+
+
+def test_log_document_responses_reports_status_and_headers():
+    logger = FakeLogger()
+
+    class Driver:
+        def get_log(self, _log_type):
+            return [
+                {
+                    "message": json.dumps(
+                        {
+                            "message": {
+                                "method": "Network.responseReceived",
+                                "params": {
+                                    "type": "Document",
+                                    "response": {
+                                        "url": "https://www.upwork.com/nx/s/find-work/best-matches",
+                                        "status": 403,
+                                        "headers": {"cf-ray": "abc-LHR", "cf-mitigated": "block"},
+                                    },
+                                },
+                            }
+                        }
+                    )
+                }
+            ]
+
+    app._log_document_responses(Driver(), logger)
+
+    message = str(logger.messages[0][1])
+    assert "403" in message
+    assert "cf-ray=abc-LHR" in message
+    assert "cf-mitigated=block" in message
+
+    app._log_document_responses(None, logger)
+
+
 def test_process_jobs_counts_insert_update_failure_and_skipped(monkeypatch):
     conn = sqlite3.connect(":memory:")
     cursor = conn.cursor()