From c2ac3bcd48b40165fef6822fdf5e47b50af25c48 Mon Sep 17 00:00:00 2001 From: roperi <9901508+roperi@users.noreply.github.com> Date: Fri, 30 Oct 2026 10:33:41 +0000 Subject: [PATCH] Detect Cloudflare block pages and pace the Best Matches feed Cloudflare's block page is served at the requested URL, so _authenticated() treated it as a completed login: the scraper scrolled the block page and then reported a successful run with zero jobs. Detect the block page explicitly, fail fast with an actionable message, and check for it before and during scrolling. Open Best Matches directly instead of the login page on every run. With a persistent profile the login page only redirects to the dashboard, and a redirect followed by a second top-level navigation within a second is scored as automation. Accept the /nx/s/ session-scoped path so an authenticated session is not navigated again. Wait for feed tiles before sending input, jitter the scroll cadence, and stop once the feed stops moving. Read job URLs from the rendered job links with the opening UID as a fallback, and fail when no job links are found instead of reporting zero processed jobs. Log the HTTP status and Cloudflare headers of each document load so a refused run can be diagnosed from the response instead of inferred from page text. --- README.md | 27 +++++ tests/test_app.py | 208 ++++++++++++++++++++++++++++++++-- tests/test_auth.py | 74 ++++++++++++- tests/test_browser.py | 5 + upwork_scraper/app.py | 228 +++++++++++++++++++++++++++++++++----- upwork_scraper/auth.py | 83 +++++++++++++- upwork_scraper/browser.py | 3 + 7 files changed, 585 insertions(+), 43 deletions(-) diff --git a/README.md b/README.md index 6a26b2c..c9d6ec0 100644 --- a/README.md +++ b/README.md @@ -158,6 +158,33 @@ check to complete before submitting each step, so no manual action is normally r browser profile usually keeps the Cloudflare clearance cookie and skips the visible check on later runs. +### Cloudflare block pages + +Occasionally Cloudflare serves its **"Sorry, you have been blocked"** / **"Attention Required!"** page +instead of Upwork. That page is a request refusal, not a login failure: the request never reaches +Upwork, so no cookie, reload, or wait inside the browser will get past it. It is Cloudflare's bot +scoring rejecting the automated browser session. The same machine and network usually still load +Upwork normally in Chrome, and a `curl` request from that machine is blocked outright, so it does not +indicate a network fault. + +The scraper detects that page and stops immediately with a message naming the block instead of +scrolling an empty feed. Every run also logs the HTTP outcome of each Upwork document load, +including the status, `cf-ray`, `cf-mitigated`, and `retry-after` values, for example: + +``` +[INFO] Response 403 for https://www.upwork.com/nx/find-work/best-matches (cf-ray=9d2f...-LHR, cf-mitigated=challenge) +[INFO] Response 200 for https://www.upwork.com/nx/s/find-work/best-matches (cf-ray=9d2f...-LHR, cf-mitigated=-) +``` + +Because each retry from the same session extends the block: + +* Do not re-run the scraper in a loop after a block. Wait several minutes and try once. +* Confirm the network is fine by opening Upwork in your normal browser. +* Reusing an aged browser profile helps: log in once manually in the scraper's visible browser and + browse a little, so the profile carries normal cookies before automated runs reuse it. +* A proxy (`UPWORK_PROXY_SERVER`) only helps when the route itself is the problem. If your normal + browser works from the same connection, changing proxies will not raise the automation score. + The browser major version is detected automatically. Users should not normally maintain a Chrome version list. If Upwork reports network restrictions, configure the same stable proxy route that works in diff --git a/tests/test_app.py b/tests/test_app.py index 5dd98dd..47c19ed 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -1,15 +1,19 @@ +import json +import logging import sqlite3 from pathlib import Path from types import SimpleNamespace from upwork_scraper import app +from upwork_scraper import auth as app_auth from upwork_scraper.browser import BrowserSpec from upwork_scraper.config import Settings from upwork_scraper.database import create_db -class FakeLogger: +class FakeLogger(logging.Logger): def __init__(self): + super().__init__("fake") self.messages = [] def info(self, *args, **kwargs): @@ -18,6 +22,9 @@ def info(self, *args, **kwargs): def warning(self, *args, **kwargs): self.messages.append(("warning", args)) + def error(self, *args, **kwargs): + self.messages.append(("error", args)) + def exception(self, *args, **kwargs): self.messages.append(("exception", args)) @@ -29,6 +36,12 @@ def __init__(self): def quit(self): self.closed = True + def execute_cdp_cmd(self, _command, _params): + return {} + + def get_log(self, _log_type): + return [] + def test_validate_login_does_not_open_database(monkeypatch): settings = Settings(username="user@example.com", password="secret", first_name="FirstName") @@ -67,14 +80,17 @@ def test_cli_converts_failure_to_one(monkeypatch): def test_job_urls_are_built_from_feed_tiles(): class Driver: def execute_script(self, _script): - return [ - { - "uid": "2105718172637605907", - "title": "PromptOps Engineer for Deterministic AI Agents", - }, - {"uid": None, "title": "Missing opening UID"}, - {"uid": "2105364230841134304", "title": "Build HIPAA-Compliant AI system"}, - ] + return { + "tiles": [ + { + "uid": "2105718172637605907", + "title": "PromptOps Engineer for Deterministic AI Agents", + }, + {"uid": None, "title": "Missing opening UID"}, + {"uid": "2105364230841134304", "title": "Build HIPAA-Compliant AI system"}, + ], + "links": [], + } assert app._job_urls(Driver()) == [ "https://www.upwork.com/jobs/PromptOps-Engineer-for-Deterministic-AI-Agents" @@ -83,6 +99,23 @@ def execute_script(self, _script): ] +def test_job_urls_fall_back_to_feed_links_when_tiles_have_no_hooks(): + class Driver: + def execute_script(self, _script): + return { + "tiles": [{"href": None, "uid": None, "title": None}], + "links": [ + "/jobs/Ops-Engineer_~022116093407321073630/?referrer_url_path=find_work_home", + "/jobs/Ops-Engineer_~022116093407321073630/?referrer_url_path=find_work_home", + "/nx/search/jobs/?ontology_skill_uid=1031626712264720384", + ], + } + + assert app._job_urls(Driver()) == [ + "https://www.upwork.com/jobs/Ops-Engineer_~022116093407321073630/" + ] + + def test_load_job_list_waits_for_stable_empty_results(monkeypatch): class Driver: def __init__(self): @@ -112,6 +145,7 @@ def until(self, condition): driver = Driver() logger = FakeLogger() monkeypatch.setattr(app, "SCROLL_STEPS", 2) + monkeypatch.setattr(app, "FEED_READY_TIMEOUT", 0) monkeypatch.setattr(app.time, "sleep", lambda _seconds: None) monkeypatch.setattr(app, "WebDriverWait", ImmediateWait) @@ -121,6 +155,162 @@ def until(self, condition): assert any("Job list loaded" in message[0] for _, message in logger.messages) +def test_wait_for_feed_returns_as_soon_as_tiles_render(monkeypatch): + class Driver: + def __init__(self): + self.tile_probes = 0 + + def execute_script(self, script): + if "job-tile" in script: + self.tile_probes += 1 + return 0 if self.tile_probes < 2 else 30 + return False + + driver = Driver() + logger = FakeLogger() + monkeypatch.setattr(app.time, "sleep", lambda _seconds: None) + + app._wait_for_feed(driver, logger) + + assert driver.tile_probes == 2 + assert not any(level == "warning" for level, _ in logger.messages) + + +def test_job_urls_prefer_the_tile_link_and_strip_tracking_parameters(): + class Driver: + def execute_script(self, _script): + return { + "tiles": [ + { + "href": "/jobs/ML-Ops-Engineer_~022116093407321073630/" + "?referrer_url_path=find_work_home", + "uid": "2116093407321073630", + "title": "ML Ops Engineer for Cloud Features", + }, + { + "href": "https://www.upwork.com/jobs/Absolute_~02200?x=1", + "uid": None, + "title": None, + }, + {"href": None, "uid": None, "title": "Unusable tile"}, + ], + "links": ["/jobs/ignored_~02999"], + } + + assert app._job_urls(Driver()) == [ + "https://www.upwork.com/jobs/ML-Ops-Engineer_~022116093407321073630/", + "https://www.upwork.com/jobs/Absolute_~02200", + ] + + +def test_load_job_list_stops_on_cloudflare_block(monkeypatch): + """A block page must end the run instead of scrolling an empty page.""" + + class Driver: + current_url = "https://www.upwork.com/nx/find-work/best-matches" + scrolls = 0 + + def execute_script(self, script, *arguments): + if script == app_auth.BLOCKED_PAGE_JS: + return True + self.scrolls += 1 + return None + + driver = Driver() + try: + app._load_job_list(driver, FakeLogger()) + except app_auth.UpworkBlockedError: + pass + else: + raise AssertionError("expected UpworkBlockedError before scrolling") + assert driver.scrolls == 0 + + +def test_document_responses_extract_cloudflare_verdicts(): + class Driver: + def get_log(self, _log_type): + return [ + { + "message": json.dumps( + { + "message": { + "method": "Network.responseReceived", + "params": { + "type": "Document", + "response": { + "url": "https://www.upwork.com/nx/s/find-work/best-matches", + "status": 403, + "headers": { + "CF-Ray": "a5296037484ded07-LHR", + "cf-mitigated": "challenge", + "Retry-After": "60", + }, + }, + }, + } + } + ) + }, + { + "message": json.dumps( + { + "message": { + "method": "Network.responseReceived", + "params": { + "type": "XHR", + "response": {"url": "https://api", "status": 200}, + }, + } + } + ) + }, + {"message": "not json"}, + ] + + documents = app._document_responses(Driver()) + + assert len(documents) == 1 + assert documents[0]["status"] == 403 + assert documents[0]["cf-ray"] == "a5296037484ded07-LHR" + assert documents[0]["cf-mitigated"] == "challenge" + assert documents[0]["retry-after"] == "60" + + +def test_log_document_responses_reports_status_and_headers(): + logger = FakeLogger() + + class Driver: + def get_log(self, _log_type): + return [ + { + "message": json.dumps( + { + "message": { + "method": "Network.responseReceived", + "params": { + "type": "Document", + "response": { + "url": "https://www.upwork.com/nx/s/find-work/best-matches", + "status": 403, + "headers": {"cf-ray": "abc-LHR", "cf-mitigated": "block"}, + }, + }, + } + } + ) + } + ] + + app._log_document_responses(Driver(), logger) + + message = str(logger.messages[0][1]) + assert "403" in message + assert "cf-ray=abc-LHR" in message + assert "cf-mitigated=block" in message + + app._log_document_responses(None, logger) + + def test_process_jobs_counts_insert_update_failure_and_skipped(monkeypatch): conn = sqlite3.connect(":memory:") cursor = conn.cursor() diff --git a/tests/test_auth.py b/tests/test_auth.py index 9dea05b..2f912bc 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -1,3 +1,5 @@ +from selenium.common.exceptions import TimeoutException + from upwork_scraper import auth from upwork_scraper.auth import ( BEST_MATCHES_URL, @@ -41,6 +43,7 @@ def __init__(self, url, buttons=None, fields=None, redirect_url=None): self.challenge = False self.tokens = [] self.body_text = "" + self.blocked = False def get(self, url): self.visited_urls.append(url) @@ -69,6 +72,8 @@ def execute_script(self, script, *_args): if isinstance(self.challenge, list): return self.challenge.pop(0) return self.challenge + if script == auth.BLOCKED_PAGE_JS: + return self.blocked return None @@ -180,7 +185,9 @@ def test_login_navigates_authenticated_profile_to_best_matches(monkeypatch): assert driver.visited_urls == [BEST_MATCHES_URL] -def test_login_handles_authenticated_redirect_from_login_url(monkeypatch): +def test_login_opens_best_matches_directly_for_a_signed_in_profile(monkeypatch): + """A persistent session must not route through the login page first.""" + driver = FakeDriver("about:blank", redirect_url=BEST_MATCHES_URL) settings = Settings(username="user@example.com", password="secret", first_name="FirstName") @@ -191,10 +198,73 @@ def unexpected_cookie_scan(_driver): auth.login(driver, settings, lambda _message: None) - assert driver.visited_urls == [LOGIN_URL] + assert driver.visited_urls == [BEST_MATCHES_URL] + + +def test_login_falls_back_to_the_login_page_when_the_feed_is_unavailable(monkeypatch): + monkeypatch.setattr(auth.time, "sleep", lambda _seconds: None) + driver = FakeDriver("about:blank") + settings = Settings( + username="user@example.com", + password="secret", + first_name="FirstName", + verification_timeout=0, + ) + + try: + auth.login(driver, settings, lambda _message: None) + except TimeoutException: + pass + else: + raise AssertionError("expected a timeout without a feed or login form") + + assert driver.visited_urls == [BEST_MATCHES_URL, LOGIN_URL] def test_login_warning_detects_security_interstitials(): assert _login_warning("We detected abnormal behavior") == "abnormal behavior" assert _login_warning("The account has unusual activity") == "unusual activity" assert _login_warning("Normal login page") is None + + +def test_at_best_matches_accepts_the_session_scoped_redirect_url(): + assert auth._at_best_matches(FakeDriver("https://www.upwork.com/nx/find-work/best-matches")) + assert auth._at_best_matches(FakeDriver("https://www.upwork.com/nx/s/find-work/best-matches")) + assert not auth._at_best_matches(FakeDriver("https://www.upwork.com/nx/find-work/home")) + + +def test_blocked_page_is_not_treated_as_authenticated(): + driver = FakeDriver(BEST_MATCHES_URL) + driver.blocked = True + + assert auth._blocked_page(driver) is True + assert _authenticated(driver) is False + + +def test_login_fails_fast_when_cloudflare_blocks_the_login_page(): + driver = FakeDriver(LOGIN_URL) + driver.blocked = True + settings = Settings(username="user@example.com", password="secret", first_name="FirstName") + + try: + auth.login(driver, settings, lambda _message: None) + except auth.UpworkBlockedError as exc: + assert "bot scoring" in str(exc) + else: + raise AssertionError("expected UpworkBlockedError while Cloudflare blocks the network") + + +def test_go_to_best_matches_fails_fast_when_cloudflare_blocks(): + class BlockingDriver(FakeDriver): + def get(self, url): + super().get(url) + self.blocked = True + + driver = BlockingDriver("https://www.upwork.com/nx/find-work/home") + + try: + auth._go_to_best_matches(driver) + except auth.UpworkBlockedError: + pass + else: + raise AssertionError("expected UpworkBlockedError from the block page") diff --git a/tests/test_browser.py b/tests/test_browser.py index d919e88..73fcc5f 100644 --- a/tests/test_browser.py +++ b/tests/test_browser.py @@ -32,10 +32,14 @@ class FakeOptions: def __init__(self): self.arguments = [] self.user_data_dir = None + self.capabilities = {} def add_argument(self, argument): self.arguments.append(argument) + def set_capability(self, name, value): + self.capabilities[name] = value + options = FakeOptions() monkeypatch.setattr(browser.uc, "ChromeOptions", lambda: options) monkeypatch.setattr(browser.uc, "Chrome", lambda **kwargs: kwargs) @@ -54,3 +58,4 @@ def add_argument(self, argument): assert options.page_load_strategy == "eager" assert options.user_data_dir == str(tmp_path / "profile") assert "--proxy-server=socks5://127.0.0.1:1080" in options.arguments + assert options.capabilities["goog:loggingPrefs"] == {"performance": "ALL"} diff --git a/upwork_scraper/app.py b/upwork_scraper/app.py index 6b19a8c..83776e0 100644 --- a/upwork_scraper/app.py +++ b/upwork_scraper/app.py @@ -2,7 +2,9 @@ from __future__ import annotations +import json import logging +import random import re import sys import time @@ -11,11 +13,12 @@ from datetime import datetime from pathlib import Path +from selenium.common.exceptions import WebDriverException from selenium.webdriver.common.by import By from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.support.ui import WebDriverWait -from .auth import login +from .auth import UpworkBlockedError, ensure_not_blocked, login from .browser import discover_browser, launch_driver from .config import ConfigurationError, load_settings, validate_proxy_server from .database import connect_to_db, create_db @@ -23,8 +26,11 @@ SCROLL_STEPS = 12 SCROLL_PAUSE_SECONDS = 0.5 +FEED_READY_TIMEOUT = 45 JOB_COUNT_STABLE_POLLS = 3 JOB_COUNT_STABLE_TIMEOUT = 30 +DOCUMENT_HEADERS = ("cf-ray", "cf-mitigated", "server", "retry-after", "location") +DOCUMENT_LOG_LIMIT = 4 @dataclass(frozen=True) @@ -57,6 +63,69 @@ def _logger(level: str) -> logging.Logger: return logger +def _enable_response_logging(driver) -> None: + """Start collecting document responses from Chrome's performance log. + + The launch options already request performance logging; enabling the + network domain makes sure every document response is reported even when a + navigation is short-circuited by a redirect or a refusal. + """ + + try: + driver.execute_cdp_cmd("Network.enable", {}) + except WebDriverException: + pass + + +def _document_responses(driver) -> list[dict]: + """Return the document responses recorded since the last call.""" + + try: + entries = driver.get_log("performance") + except WebDriverException: + return [] + + documents = [] + for entry in entries: + try: + message = json.loads(entry["message"])["message"] + except (KeyError, TypeError, ValueError): + continue + if message.get("method") != "Network.responseReceived": + continue + params = message.get("params") or {} + if params.get("type") != "Document": + continue + response = params.get("response") or {} + headers = {key.lower(): value for key, value in (response.get("headers") or {}).items()} + document = { + "url": response.get("url", ""), + "status": response.get("status"), + } + for header in DOCUMENT_HEADERS: + document[header] = headers.get(header) + documents.append(document) + return documents + + +def _log_document_responses(driver, logger: logging.Logger) -> None: + """Log the HTTP outcome of each Upwork document load. + + Cloudflare reports its verdict in the response, so these lines say whether + a run was refused, challenged, or served, without guessing from page text. + """ + + if driver is None: + return + for document in _document_responses(driver)[-DOCUMENT_LOG_LIMIT:]: + details = ", ".join( + f"{header}={document.get(header) or '-'}" + for header in DOCUMENT_HEADERS + if header in {"cf-ray", "cf-mitigated", "retry-after"} + ) + logger.info("Response %s for %s (%s)", document["status"], document["url"], details) + + def _arguments(argv: list[str] | None): parser = ArgumentParser(description="Scrape Upwork Best Matches once") parser.add_argument("--check-config", action="store_true", help="validate settings and browser") @@ -73,34 +142,70 @@ def _arguments(argv: list[str] | None): return parser.parse_args(argv) +_JOB_TILES_JS = """ +const tiles = Array.from(document.querySelectorAll('[data-test="job-tile"], .job-tile')); +return { + tiles: tiles.map(tile => { + const link = tile.querySelector('a[href*="/jobs/"]'); + const tracker = tile.querySelector('[data-ev-opening_uid]'); + const title = tile.querySelector('[data-test="job-title"]'); + return { + href: link ? link.getAttribute('href') : null, + uid: tracker ? tracker.getAttribute('data-ev-opening_uid') : null, + title: title ? title.textContent.trim() : null, + }; + }), + // Fallback for variants whose tiles carry no data-test/class hooks: the job + // links themselves are the canonical URLs and are always rendered. + links: Array.from(document.querySelectorAll('a[href*="/jobs/"]')) + .map(anchor => anchor.getAttribute('href')) + .filter(href => href && /_~02\\d+/.test(href)), +}; +""" + + def _job_urls(driver) -> list[str]: - """Build canonical job URLs from the feed tiles. + """Collect canonical job URLs from the feed, in document order.""" - Upwork no longer links job titles to ``/jobs/`` pages. Each tile still - carries the opening UID, and the job cipher is ``~02`` followed by it. - """ + found = driver.execute_script(_JOB_TILES_JS) or {} + urls = [ + url for url in (_tile_job_url(tile) for tile in found.get("tiles", [])) if url is not None + ] + if urls: + return urls + + fallback: list[str] = [] + for href in found.get("links", []): + url = _absolute_job_url(href) + if url is not None and url not in fallback: + fallback.append(url) + return fallback + + +def _absolute_job_url(href: str) -> str | None: + """Normalize a rendered job link to a canonical absolute URL.""" + + href = (href or "").strip() + if href.startswith("/jobs/"): + return f"https://www.upwork.com{href.split('?', 1)[0].split('#', 1)[0]}" + if href.startswith("https://www.upwork.com/jobs/"): + return href.split("?", 1)[0].split("#", 1)[0] + return None - tiles = driver.execute_script( - """ - return Array.from(document.querySelectorAll('[data-test="job-tile"]')).map(tile => { - const tracker = tile.querySelector('[data-ev-opening_uid]'); - const title = tile.querySelector('[data-test="job-title"]'); - if (!tracker || !title) return null; - return { - uid: tracker.getAttribute('data-ev-opening_uid'), - title: title.textContent.trim(), - }; - }).filter(Boolean); - """ - ) - urls = [] - for tile in tiles: - uid = tile.get("uid") - if not uid: - continue - slug = re.sub(r"[^A-Za-z0-9]+", "-", tile.get("title") or "").strip("-") - urls.append(f"https://www.upwork.com/jobs/{slug}_~02{uid}") - return urls + +def _tile_job_url(tile: dict) -> str | None: + """Return the canonical job URL for one feed tile, or ``None``.""" + + url = _absolute_job_url(tile.get("href") or "") + if url is not None: + return url + + uid = tile.get("uid") + title = tile.get("title") + if not uid or not title: + return None + slug = re.sub(r"[^A-Za-z0-9]+", "-", title).strip("-") + return f"https://www.upwork.com/jobs/{slug}_~02{uid}" def _feed_container(driver): @@ -143,23 +248,69 @@ def _scroll_offset(driver, container) -> int: return driver.execute_script("return Math.round(arguments[0].scrollTop);", container) +def _job_tile_count(driver) -> int: + """Count rendered feed entries without parsing their contents.""" + + count = driver.execute_script( + """ + const tiles = document.querySelectorAll('[data-test="job-tile"], .job-tile').length; + if (tiles) return tiles; + return Array.from(document.querySelectorAll('a[href*="/jobs/"]')) + .filter(anchor => /_~02\\d+/.test(anchor.getAttribute('href') || '')).length; + """ + ) + return count if isinstance(count, int) else 0 + + +def _wait_for_feed(driver, logger: logging.Logger) -> None: + """Wait for the feed to render before sending any input. + + Sending wheel events into the blank shell that loads first is both useless + (there is nothing to scroll) and a strong automation signal, so wait for + the first tile instead of scrolling on a timer. + """ + + deadline = time.monotonic() + FEED_READY_TIMEOUT + while time.monotonic() < deadline: + ensure_not_blocked(driver) + if _job_tile_count(driver): + return + time.sleep(0.5) + logger.warning("No job tiles rendered after %d seconds; scrolling anyway", FEED_READY_TIMEOUT) + + def _load_job_list(driver, logger: logging.Logger) -> None: """Scroll through results and wait briefly for lazy-loaded jobs to settle.""" + _wait_for_feed(driver, logger) + + previous_offset = -1 + stalled = 0 for step in range(1, SCROLL_STEPS + 1): + # Cloudflare can swap in its block page after an initial success, so + # check every step instead of scrolling twenty seconds of nothing. + ensure_not_blocked(driver) # The feed renders asynchronously, so the scroll container may not # exist yet on the first steps. Rediscover it every step instead of # caching a None/decoy and silently no-op'ing the rest of the run. container = _feed_container(driver) _scroll_feed(driver, container) - time.sleep(SCROLL_PAUSE_SECONDS) + time.sleep(random.uniform(SCROLL_PAUSE_SECONDS, SCROLL_PAUSE_SECONDS * 2)) + offset = _scroll_offset(driver, container) logger.info( "Loading jobs: scroll %d/%d; %d job links visible; offset %d", step, SCROLL_STEPS, len(_job_urls(driver)), - _scroll_offset(driver, container), + offset, ) + # A wheel event on an already-scrolled-to-bottom feed is pure synthetic + # input, so stop once two consecutive steps move nothing. + stalled = stalled + 1 if offset == previous_offset else 0 + previous_offset = offset + if stalled >= 2: + logger.info("Feed reached the bottom after %d scrolls", step) + break container = _feed_container(driver) if container is not None: @@ -285,11 +436,18 @@ def main(argv: list[str] | None = None) -> bool: try: driver, browser = launch_driver(settings) logger.info("Using %s (%s)", browser.executable, browser.version) + _enable_response_logging(driver) logger.info("Validating Upwork login") login(driver, settings, lambda message: logger.warning(message)) + _log_document_responses(driver, logger) logger.info("Upwork login validation succeeded") return True + except UpworkBlockedError as exc: + _log_document_responses(driver, logger) + logger.error("%s", exc) + return False except Exception: + _log_document_responses(driver, logger) logger.exception("Login validation failed") return False finally: @@ -305,9 +463,11 @@ def main(argv: list[str] | None = None) -> bool: create_db(conn, cursor) driver, browser = launch_driver(settings) logger.info("Using %s (%s)", browser.executable, browser.version) + _enable_response_logging(driver) logger.info("Checking Upwork authentication") login(driver, settings, lambda message: logger.warning(message)) logger.info("Upwork authentication ready") + _log_document_responses(driver, logger) _load_job_list(driver, logger) @@ -320,6 +480,13 @@ def main(argv: list[str] | None = None) -> bool: job_posts = text.split("Posted")[1:] job_urls = _job_urls(driver) logger.info("Parsed %d job posts and found %d job links", len(job_posts), len(job_urls)) + if not job_urls: + # Reporting success here would hide both a block page and a changed + # feed markup behind "0 jobs processed". + raise RuntimeError( + "No job links were found on the Best Matches feed; the page markup may have " + "changed, or Upwork served an unexpected page" + ) counts = _process_jobs(job_posts, job_urls, cursor, logger) conn.commit() logger.info( @@ -333,7 +500,12 @@ def main(argv: list[str] | None = None) -> bool: counts.skipped, ) return True + except UpworkBlockedError as exc: + _log_document_responses(driver, logger) + logger.error("%s", exc) + return False except Exception: + _log_document_responses(driver, logger) logger.exception("Scraping failed") return False finally: diff --git a/upwork_scraper/auth.py b/upwork_scraper/auth.py index 3e9bd72..5aaab37 100644 --- a/upwork_scraper/auth.py +++ b/upwork_scraper/auth.py @@ -40,6 +40,58 @@ """ +BLOCKED_PAGE_JS = """ +const title = (document.title || '').toLowerCase(); +if (title.includes('attention required') || title.includes('access denied')) return true; +// Cloudflare's block page is a few kilobytes of plain HTML. Bail out on large +// documents first so checking a loaded feed does not pay for a layout pass. +const text = ((document.body && document.body.textContent) || '').toLowerCase(); +if (text.length > 50000) return false; +return text.includes('you have been blocked') + || text.includes('unable to access upwork.com') + || text.includes('why have i been blocked'); +""" +BLOCKED_MESSAGE = ( + "Cloudflare refused to pass {url} through to Upwork and returned its " + '"You have been blocked" page. The request never reached Upwork, so this is bot ' + "scoring rejecting this automated browser session, not a login or page problem. " + "Do not retry immediately: each attempt extends the block. Wait several minutes, " + "then run once. If Upwork loads normally in your own Chrome on this machine, the " + "network is fine and only the automation session is being penalised." +) + + +class UpworkBlockedError(RuntimeError): + """Raised when Cloudflare refuses to serve Upwork to the automated browser.""" + + +def _blocked_page(driver) -> bool: + """Report whether Upwork served Cloudflare's hard block page. + + Unlike an interstitial challenge, this page cannot clear itself: the request + never reached Upwork, so no widget, cookie, or reload will recover it. + """ + + try: + return bool(driver.execute_script(BLOCKED_PAGE_JS)) + except InvalidSessionIdException: + raise + except WebDriverException: + # A navigation can destroy the execution context mid-check. + return False + + +def ensure_not_blocked(driver) -> None: + """Fail fast with an actionable message instead of scraping a block page.""" + + if _blocked_page(driver): + try: + url = driver.current_url + except WebDriverException: + url = BEST_MATCHES_URL + raise UpworkBlockedError(BLOCKED_MESSAGE.format(url=url)) + + def _visible_button(driver, button_ids=()): for button_id in button_ids: button = _visible_element_by_id(driver, button_id) @@ -139,6 +191,7 @@ def _wait_for_login_field(driver, timeout: int, logger): deadline = time.monotonic() + timeout challenge_reported = False while time.monotonic() < deadline: + ensure_not_blocked(driver) if _authenticated(driver): return True if _cloudflare_challenge(driver): @@ -214,6 +267,10 @@ def _click_submit(driver, field, button_ids, timeout: int, logger) -> None: def _authenticated(driver) -> bool: try: + if _blocked_page(driver): + # The block page is not a login page, but treating it as an + # authenticated session makes the scraper silently scrape it. + return False parsed_url = urlsplit(driver.current_url) hostname = parsed_url.hostname or "" if hostname != "upwork.com" and not hostname.endswith(".upwork.com"): @@ -228,13 +285,18 @@ def _authenticated(driver) -> bool: def _at_best_matches(driver) -> bool: - parsed_url = urlsplit(driver.current_url) - return parsed_url.path.rstrip("/") == "/nx/find-work/best-matches" + # Upwork redirects the public path to its session-scoped counterpart + # (/nx/s/...), so both forms mean "already on Best Matches". + path = urlsplit(driver.current_url).path.rstrip("/") + return path in ("/nx/find-work/best-matches", "/nx/s/find-work/best-matches") def _go_to_best_matches(driver) -> None: if not _at_best_matches(driver): driver.get(BEST_MATCHES_URL) + # A 403 block page is served at the requested URL, so the address alone + # cannot tell a loaded feed from a refusal. + ensure_not_blocked(driver) def _visible_element_by_id(driver, element_id: str): @@ -270,8 +332,20 @@ def login(driver, settings: Settings, logger: Callable[[str], None]) -> None: _go_to_best_matches(driver) return - driver.get(LOGIN_URL) - login_step = _wait_for_login_field(driver, settings.verification_timeout, logger) + # Open the target page directly. A persistent profile is normally still + # signed in, and routing through the login page makes the browser follow a + # redirect and then start a second top-level navigation within a second, + # which Cloudflare scores as automation. An expired session simply + # redirects here to the login form, where the flow below continues. + driver.get(BEST_MATCHES_URL) + ensure_not_blocked(driver) + try: + login_step = _wait_for_login_field(driver, settings.verification_timeout, logger) + except TimeoutException: + # Neither the feed nor a login form came back; retry on the login page. + driver.get(LOGIN_URL) + login_step = _wait_for_login_field(driver, settings.verification_timeout, logger) + if login_step is True: _go_to_best_matches(driver) return @@ -328,6 +402,7 @@ def login(driver, settings: Settings, logger: Callable[[str], None]) -> None: reported_warning: str | None = None challenge_reported = False while time.monotonic() < deadline: + ensure_not_blocked(driver) if _authenticated(driver): _go_to_best_matches(driver) return diff --git a/upwork_scraper/browser.py b/upwork_scraper/browser.py index 3d58455..0f58c3f 100644 --- a/upwork_scraper/browser.py +++ b/upwork_scraper/browser.py @@ -81,6 +81,9 @@ def _launch_with_driver(settings: Settings, spec: BrowserSpec, driver_path: Path options.headless = False options.page_load_strategy = "eager" options.add_argument("--disable-dev-shm-usage") + # Lets the app report the HTTP outcome of each Upwork document load + # (status, cf-ray, cf-mitigated) so block pages can be diagnosed. + options.set_capability("goog:loggingPrefs", {"performance": "ALL"}) options.user_data_dir = str(settings.browser_profile_dir) if settings.proxy_server: options.add_argument(f"--proxy-server={settings.proxy_server}")