diff --git a/README.md b/README.md index 6a26b2c..c9d6ec0 100644 --- a/README.md +++ b/README.md @@ -158,6 +158,33 @@ check to complete before submitting each step, so no manual action is normally r browser profile usually keeps the Cloudflare clearance cookie and skips the visible check on later runs. +### Cloudflare block pages + +Occasionally Cloudflare serves its **"Sorry, you have been blocked"** / **"Attention Required!"** page +instead of Upwork. That page is a request refusal, not a login failure: the request never reaches +Upwork, so no cookie, reload, or wait inside the browser will get past it. It is Cloudflare's bot +scoring rejecting the automated browser session. The same machine and network usually still load +Upwork normally in Chrome, and a `curl` request from that machine is blocked outright, so it does not +indicate a network fault. + +The scraper detects that page and stops immediately with a message naming the block instead of +scrolling an empty feed. Every run also logs the HTTP outcome of each Upwork document load, +including the status, `cf-ray`, `cf-mitigated`, and `retry-after` values, for example: + +``` +[INFO] Response 403 for https://www.upwork.com/nx/find-work/best-matches (cf-ray=9d2f...-LHR, cf-mitigated=challenge) +[INFO] Response 200 for https://www.upwork.com/nx/s/find-work/best-matches (cf-ray=9d2f...-LHR, cf-mitigated=-) +``` + +Because each retry from the same session extends the block: + +* Do not re-run the scraper in a loop after a block. Wait several minutes and try once. +* Confirm the network is fine by opening Upwork in your normal browser. +* Reusing an aged browser profile helps: log in once manually in the scraper's visible browser and + browse a little, so the profile carries normal cookies before automated runs reuse it. +* A proxy (`UPWORK_PROXY_SERVER`) only helps when the route itself is the problem. If your normal + browser works from the same connection, changing proxies will not raise the automation score. + The browser major version is detected automatically. Users should not normally maintain a Chrome version list. If Upwork reports network restrictions, configure the same stable proxy route that works in diff --git a/tests/test_app.py b/tests/test_app.py index 5dd98dd..47c19ed 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -1,15 +1,19 @@ +import json +import logging import sqlite3 from pathlib import Path from types import SimpleNamespace from upwork_scraper import app +from upwork_scraper import auth as app_auth from upwork_scraper.browser import BrowserSpec from upwork_scraper.config import Settings from upwork_scraper.database import create_db -class FakeLogger: +class FakeLogger(logging.Logger): def __init__(self): + super().__init__("fake") self.messages = [] def info(self, *args, **kwargs): @@ -18,6 +22,9 @@ def info(self, *args, **kwargs): def warning(self, *args, **kwargs): self.messages.append(("warning", args)) + def error(self, *args, **kwargs): + self.messages.append(("error", args)) + def exception(self, *args, **kwargs): self.messages.append(("exception", args)) @@ -29,6 +36,12 @@ def __init__(self): def quit(self): self.closed = True + def execute_cdp_cmd(self, _command, _params): + return {} + + def get_log(self, _log_type): + return [] + def test_validate_login_does_not_open_database(monkeypatch): settings = Settings(username="user@example.com", password="secret", first_name="FirstName") @@ -67,14 +80,17 @@ def test_cli_converts_failure_to_one(monkeypatch): def test_job_urls_are_built_from_feed_tiles(): class Driver: def execute_script(self, _script): - return [ - { - "uid": "2105718172637605907", - "title": "PromptOps Engineer for Deterministic AI Agents", - }, - {"uid": None, "title": "Missing opening UID"}, - {"uid": "2105364230841134304", "title": "Build HIPAA-Compliant AI system"}, - ] + return { + "tiles": [ + { + "uid": "2105718172637605907", + "title": "PromptOps Engineer for Deterministic AI Agents", + }, + {"uid": None, "title": "Missing opening UID"}, + {"uid": "2105364230841134304", "title": "Build HIPAA-Compliant AI system"}, + ], + "links": [], + } assert app._job_urls(Driver()) == [ "https://www.upwork.com/jobs/PromptOps-Engineer-for-Deterministic-AI-Agents" @@ -83,6 +99,23 @@ def execute_script(self, _script): ] +def test_job_urls_fall_back_to_feed_links_when_tiles_have_no_hooks(): + class Driver: + def execute_script(self, _script): + return { + "tiles": [{"href": None, "uid": None, "title": None}], + "links": [ + "/jobs/Ops-Engineer_~022116093407321073630/?referrer_url_path=find_work_home", + "/jobs/Ops-Engineer_~022116093407321073630/?referrer_url_path=find_work_home", + "/nx/search/jobs/?ontology_skill_uid=1031626712264720384", + ], + } + + assert app._job_urls(Driver()) == [ + "https://www.upwork.com/jobs/Ops-Engineer_~022116093407321073630/" + ] + + def test_load_job_list_waits_for_stable_empty_results(monkeypatch): class Driver: def __init__(self): @@ -112,6 +145,7 @@ def until(self, condition): driver = Driver() logger = FakeLogger() monkeypatch.setattr(app, "SCROLL_STEPS", 2) + monkeypatch.setattr(app, "FEED_READY_TIMEOUT", 0) monkeypatch.setattr(app.time, "sleep", lambda _seconds: None) monkeypatch.setattr(app, "WebDriverWait", ImmediateWait) @@ -121,6 +155,162 @@ def until(self, condition): assert any("Job list loaded" in message[0] for _, message in logger.messages) +def test_wait_for_feed_returns_as_soon_as_tiles_render(monkeypatch): + class Driver: + def __init__(self): + self.tile_probes = 0 + + def execute_script(self, script): + if "job-tile" in script: + self.tile_probes += 1 + return 0 if self.tile_probes < 2 else 30 + return False + + driver = Driver() + logger = FakeLogger() + monkeypatch.setattr(app.time, "sleep", lambda _seconds: None) + + app._wait_for_feed(driver, logger) + + assert driver.tile_probes == 2 + assert not any(level == "warning" for level, _ in logger.messages) + + +def test_job_urls_prefer_the_tile_link_and_strip_tracking_parameters(): + class Driver: + def execute_script(self, _script): + return { + "tiles": [ + { + "href": "/jobs/ML-Ops-Engineer_~022116093407321073630/" + "?referrer_url_path=find_work_home", + "uid": "2116093407321073630", + "title": "ML Ops Engineer for Cloud Features", + }, + { + "href": "https://www.upwork.com/jobs/Absolute_~02200?x=1", + "uid": None, + "title": None, + }, + {"href": None, "uid": None, "title": "Unusable tile"}, + ], + "links": ["/jobs/ignored_~02999"], + } + + assert app._job_urls(Driver()) == [ + "https://www.upwork.com/jobs/ML-Ops-Engineer_~022116093407321073630/", + "https://www.upwork.com/jobs/Absolute_~02200", + ] + + +def test_load_job_list_stops_on_cloudflare_block(monkeypatch): + """A block page must end the run instead of scrolling an empty page.""" + + class Driver: + current_url = "https://www.upwork.com/nx/find-work/best-matches" + scrolls = 0 + + def execute_script(self, script, *arguments): + if script == app_auth.BLOCKED_PAGE_JS: + return True + self.scrolls += 1 + return None + + driver = Driver() + try: + app._load_job_list(driver, FakeLogger()) + except app_auth.UpworkBlockedError: + pass + else: + raise AssertionError("expected UpworkBlockedError before scrolling") + assert driver.scrolls == 0 + + +def test_document_responses_extract_cloudflare_verdicts(): + class Driver: + def get_log(self, _log_type): + return [ + { + "message": json.dumps( + { + "message": { + "method": "Network.responseReceived", + "params": { + "type": "Document", + "response": { + "url": "https://www.upwork.com/nx/s/find-work/best-matches", + "status": 403, + "headers": { + "CF-Ray": "a5296037484ded07-LHR", + "cf-mitigated": "challenge", + "Retry-After": "60", + }, + }, + }, + } + } + ) + }, + { + "message": json.dumps( + { + "message": { + "method": "Network.responseReceived", + "params": { + "type": "XHR", + "response": {"url": "https://api", "status": 200}, + }, + } + } + ) + }, + {"message": "not json"}, + ] + + documents = app._document_responses(Driver()) + + assert len(documents) == 1 + assert documents[0]["status"] == 403 + assert documents[0]["cf-ray"] == "a5296037484ded07-LHR" + assert documents[0]["cf-mitigated"] == "challenge" + assert documents[0]["retry-after"] == "60" + + +def test_log_document_responses_reports_status_and_headers(): + logger = FakeLogger() + + class Driver: + def get_log(self, _log_type): + return [ + { + "message": json.dumps( + { + "message": { + "method": "Network.responseReceived", + "params": { + "type": "Document", + "response": { + "url": "https://www.upwork.com/nx/s/find-work/best-matches", + "status": 403, + "headers": {"cf-ray": "abc-LHR", "cf-mitigated": "block"}, + }, + }, + } + } + ) + } + ] + + app._log_document_responses(Driver(), logger) + + message = str(logger.messages[0][1]) + assert "403" in message + assert "cf-ray=abc-LHR" in message + assert "cf-mitigated=block" in message + + app._log_document_responses(None, logger) + + def test_process_jobs_counts_insert_update_failure_and_skipped(monkeypatch): conn = sqlite3.connect(":memory:") cursor = conn.cursor() diff --git a/tests/test_auth.py b/tests/test_auth.py index 9dea05b..2f912bc 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -1,3 +1,5 @@ +from selenium.common.exceptions import TimeoutException + from upwork_scraper import auth from upwork_scraper.auth import ( BEST_MATCHES_URL, @@ -41,6 +43,7 @@ def __init__(self, url, buttons=None, fields=None, redirect_url=None): self.challenge = False self.tokens = [] self.body_text = "" + self.blocked = False def get(self, url): self.visited_urls.append(url) @@ -69,6 +72,8 @@ def execute_script(self, script, *_args): if isinstance(self.challenge, list): return self.challenge.pop(0) return self.challenge + if script == auth.BLOCKED_PAGE_JS: + return self.blocked return None @@ -180,7 +185,9 @@ def test_login_navigates_authenticated_profile_to_best_matches(monkeypatch): assert driver.visited_urls == [BEST_MATCHES_URL] -def test_login_handles_authenticated_redirect_from_login_url(monkeypatch): +def test_login_opens_best_matches_directly_for_a_signed_in_profile(monkeypatch): + """A persistent session must not route through the login page first.""" + driver = FakeDriver("about:blank", redirect_url=BEST_MATCHES_URL) settings = Settings(username="user@example.com", password="secret", first_name="FirstName") @@ -191,10 +198,73 @@ def unexpected_cookie_scan(_driver): auth.login(driver, settings, lambda _message: None) - assert driver.visited_urls == [LOGIN_URL] + assert driver.visited_urls == [BEST_MATCHES_URL] + + +def test_login_falls_back_to_the_login_page_when_the_feed_is_unavailable(monkeypatch): + monkeypatch.setattr(auth.time, "sleep", lambda _seconds: None) + driver = FakeDriver("about:blank") + settings = Settings( + username="user@example.com", + password="secret", + first_name="FirstName", + verification_timeout=0, + ) + + try: + auth.login(driver, settings, lambda _message: None) + except TimeoutException: + pass + else: + raise AssertionError("expected a timeout without a feed or login form") + + assert driver.visited_urls == [BEST_MATCHES_URL, LOGIN_URL] def test_login_warning_detects_security_interstitials(): assert _login_warning("We detected abnormal behavior") == "abnormal behavior" assert _login_warning("The account has unusual activity") == "unusual activity" assert _login_warning("Normal login page") is None + + +def test_at_best_matches_accepts_the_session_scoped_redirect_url(): + assert auth._at_best_matches(FakeDriver("https://www.upwork.com/nx/find-work/best-matches")) + assert auth._at_best_matches(FakeDriver("https://www.upwork.com/nx/s/find-work/best-matches")) + assert not auth._at_best_matches(FakeDriver("https://www.upwork.com/nx/find-work/home")) + + +def test_blocked_page_is_not_treated_as_authenticated(): + driver = FakeDriver(BEST_MATCHES_URL) + driver.blocked = True + + assert auth._blocked_page(driver) is True + assert _authenticated(driver) is False + + +def test_login_fails_fast_when_cloudflare_blocks_the_login_page(): + driver = FakeDriver(LOGIN_URL) + driver.blocked = True + settings = Settings(username="user@example.com", password="secret", first_name="FirstName") + + try: + auth.login(driver, settings, lambda _message: None) + except auth.UpworkBlockedError as exc: + assert "bot scoring" in str(exc) + else: + raise AssertionError("expected UpworkBlockedError while Cloudflare blocks the network") + + +def test_go_to_best_matches_fails_fast_when_cloudflare_blocks(): + class BlockingDriver(FakeDriver): + def get(self, url): + super().get(url) + self.blocked = True + + driver = BlockingDriver("https://www.upwork.com/nx/find-work/home") + + try: + auth._go_to_best_matches(driver) + except auth.UpworkBlockedError: + pass + else: + raise AssertionError("expected UpworkBlockedError from the block page") diff --git a/tests/test_browser.py b/tests/test_browser.py index d919e88..73fcc5f 100644 --- a/tests/test_browser.py +++ b/tests/test_browser.py @@ -32,10 +32,14 @@ class FakeOptions: def __init__(self): self.arguments = [] self.user_data_dir = None + self.capabilities = {} def add_argument(self, argument): self.arguments.append(argument) + def set_capability(self, name, value): + self.capabilities[name] = value + options = FakeOptions() monkeypatch.setattr(browser.uc, "ChromeOptions", lambda: options) monkeypatch.setattr(browser.uc, "Chrome", lambda **kwargs: kwargs) @@ -54,3 +58,4 @@ def add_argument(self, argument): assert options.page_load_strategy == "eager" assert options.user_data_dir == str(tmp_path / "profile") assert "--proxy-server=socks5://127.0.0.1:1080" in options.arguments + assert options.capabilities["goog:loggingPrefs"] == {"performance": "ALL"} diff --git a/upwork_scraper/app.py b/upwork_scraper/app.py index 6b19a8c..83776e0 100644 --- a/upwork_scraper/app.py +++ b/upwork_scraper/app.py @@ -2,7 +2,9 @@ from __future__ import annotations +import json import logging +import random import re import sys import time @@ -11,11 +13,12 @@ from datetime import datetime from pathlib import Path +from selenium.common.exceptions import WebDriverException from selenium.webdriver.common.by import By from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.support.ui import WebDriverWait -from .auth import login +from .auth import UpworkBlockedError, ensure_not_blocked, login from .browser import discover_browser, launch_driver from .config import ConfigurationError, load_settings, validate_proxy_server from .database import connect_to_db, create_db @@ -23,8 +26,11 @@ SCROLL_STEPS = 12 SCROLL_PAUSE_SECONDS = 0.5 +FEED_READY_TIMEOUT = 45 JOB_COUNT_STABLE_POLLS = 3 JOB_COUNT_STABLE_TIMEOUT = 30 +DOCUMENT_HEADERS = ("cf-ray", "cf-mitigated", "server", "retry-after", "location") +DOCUMENT_LOG_LIMIT = 4 @dataclass(frozen=True) @@ -57,6 +63,69 @@ def _logger(level: str) -> logging.Logger: return logger +def _enable_response_logging(driver) -> None: + """Start collecting document responses from Chrome's performance log. + + The launch options already request performance logging; enabling the + network domain makes sure every document response is reported even when a + navigation is short-circuited by a redirect or a refusal. + """ + + try: + driver.execute_cdp_cmd("Network.enable", {}) + except WebDriverException: + pass + + +def _document_responses(driver) -> list[dict]: + """Return the document responses recorded since the last call.""" + + try: + entries = driver.get_log("performance") + except WebDriverException: + return [] + + documents = [] + for entry in entries: + try: + message = json.loads(entry["message"])["message"] + except (KeyError, TypeError, ValueError): + continue + if message.get("method") != "Network.responseReceived": + continue + params = message.get("params") or {} + if params.get("type") != "Document": + continue + response = params.get("response") or {} + headers = {key.lower(): value for key, value in (response.get("headers") or {}).items()} + document = { + "url": response.get("url", ""), + "status": response.get("status"), + } + for header in DOCUMENT_HEADERS: + document[header] = headers.get(header) + documents.append(document) + return documents + + +def _log_document_responses(driver, logger: logging.Logger) -> None: + """Log the HTTP outcome of each Upwork document load. + + Cloudflare reports its verdict in the response, so these lines say whether + a run was refused, challenged, or served, without guessing from page text. + """ + + if driver is None: + return + for document in _document_responses(driver)[-DOCUMENT_LOG_LIMIT:]: + details = ", ".join( + f"{header}={document.get(header) or '-'}" + for header in DOCUMENT_HEADERS + if header in {"cf-ray", "cf-mitigated", "retry-after"} + ) + logger.info("Response %s for %s (%s)", document["status"], document["url"], details) + + def _arguments(argv: list[str] | None): parser = ArgumentParser(description="Scrape Upwork Best Matches once") parser.add_argument("--check-config", action="store_true", help="validate settings and browser") @@ -73,34 +142,70 @@ def _arguments(argv: list[str] | None): return parser.parse_args(argv) +_JOB_TILES_JS = """ +const tiles = Array.from(document.querySelectorAll('[data-test="job-tile"], .job-tile')); +return { + tiles: tiles.map(tile => { + const link = tile.querySelector('a[href*="/jobs/"]'); + const tracker = tile.querySelector('[data-ev-opening_uid]'); + const title = tile.querySelector('[data-test="job-title"]'); + return { + href: link ? link.getAttribute('href') : null, + uid: tracker ? tracker.getAttribute('data-ev-opening_uid') : null, + title: title ? title.textContent.trim() : null, + }; + }), + // Fallback for variants whose tiles carry no data-test/class hooks: the job + // links themselves are the canonical URLs and are always rendered. + links: Array.from(document.querySelectorAll('a[href*="/jobs/"]')) + .map(anchor => anchor.getAttribute('href')) + .filter(href => href && /_~02\\d+/.test(href)), +}; +""" + + def _job_urls(driver) -> list[str]: - """Build canonical job URLs from the feed tiles. + """Collect canonical job URLs from the feed, in document order.""" - Upwork no longer links job titles to ``/jobs/`` pages. Each tile still - carries the opening UID, and the job cipher is ``~02`` followed by it. - """ + found = driver.execute_script(_JOB_TILES_JS) or {} + urls = [ + url for url in (_tile_job_url(tile) for tile in found.get("tiles", [])) if url is not None + ] + if urls: + return urls + + fallback: list[str] = [] + for href in found.get("links", []): + url = _absolute_job_url(href) + if url is not None and url not in fallback: + fallback.append(url) + return fallback + + +def _absolute_job_url(href: str) -> str | None: + """Normalize a rendered job link to a canonical absolute URL.""" + + href = (href or "").strip() + if href.startswith("/jobs/"): + return f"https://www.upwork.com{href.split('?', 1)[0].split('#', 1)[0]}" + if href.startswith("https://www.upwork.com/jobs/"): + return href.split("?", 1)[0].split("#", 1)[0] + return None - tiles = driver.execute_script( - """ - return Array.from(document.querySelectorAll('[data-test="job-tile"]')).map(tile => { - const tracker = tile.querySelector('[data-ev-opening_uid]'); - const title = tile.querySelector('[data-test="job-title"]'); - if (!tracker || !title) return null; - return { - uid: tracker.getAttribute('data-ev-opening_uid'), - title: title.textContent.trim(), - }; - }).filter(Boolean); - """ - ) - urls = [] - for tile in tiles: - uid = tile.get("uid") - if not uid: - continue - slug = re.sub(r"[^A-Za-z0-9]+", "-", tile.get("title") or "").strip("-") - urls.append(f"https://www.upwork.com/jobs/{slug}_~02{uid}") - return urls + +def _tile_job_url(tile: dict) -> str | None: + """Return the canonical job URL for one feed tile, or ``None``.""" + + url = _absolute_job_url(tile.get("href") or "") + if url is not None: + return url + + uid = tile.get("uid") + title = tile.get("title") + if not uid or not title: + return None + slug = re.sub(r"[^A-Za-z0-9]+", "-", title).strip("-") + return f"https://www.upwork.com/jobs/{slug}_~02{uid}" def _feed_container(driver): @@ -143,23 +248,69 @@ def _scroll_offset(driver, container) -> int: return driver.execute_script("return Math.round(arguments[0].scrollTop);", container) +def _job_tile_count(driver) -> int: + """Count rendered feed entries without parsing their contents.""" + + count = driver.execute_script( + """ + const tiles = document.querySelectorAll('[data-test="job-tile"], .job-tile').length; + if (tiles) return tiles; + return Array.from(document.querySelectorAll('a[href*="/jobs/"]')) + .filter(anchor => /_~02\\d+/.test(anchor.getAttribute('href') || '')).length; + """ + ) + return count if isinstance(count, int) else 0 + + +def _wait_for_feed(driver, logger: logging.Logger) -> None: + """Wait for the feed to render before sending any input. + + Sending wheel events into the blank shell that loads first is both useless + (there is nothing to scroll) and a strong automation signal, so wait for + the first tile instead of scrolling on a timer. + """ + + deadline = time.monotonic() + FEED_READY_TIMEOUT + while time.monotonic() < deadline: + ensure_not_blocked(driver) + if _job_tile_count(driver): + return + time.sleep(0.5) + logger.warning("No job tiles rendered after %d seconds; scrolling anyway", FEED_READY_TIMEOUT) + + def _load_job_list(driver, logger: logging.Logger) -> None: """Scroll through results and wait briefly for lazy-loaded jobs to settle.""" + _wait_for_feed(driver, logger) + + previous_offset = -1 + stalled = 0 for step in range(1, SCROLL_STEPS + 1): + # Cloudflare can swap in its block page after an initial success, so + # check every step instead of scrolling twenty seconds of nothing. + ensure_not_blocked(driver) # The feed renders asynchronously, so the scroll container may not # exist yet on the first steps. Rediscover it every step instead of # caching a None/decoy and silently no-op'ing the rest of the run. container = _feed_container(driver) _scroll_feed(driver, container) - time.sleep(SCROLL_PAUSE_SECONDS) + time.sleep(random.uniform(SCROLL_PAUSE_SECONDS, SCROLL_PAUSE_SECONDS * 2)) + offset = _scroll_offset(driver, container) logger.info( "Loading jobs: scroll %d/%d; %d job links visible; offset %d", step, SCROLL_STEPS, len(_job_urls(driver)), - _scroll_offset(driver, container), + offset, ) + # A wheel event on an already-scrolled-to-bottom feed is pure synthetic + # input, so stop once two consecutive steps move nothing. + stalled = stalled + 1 if offset == previous_offset else 0 + previous_offset = offset + if stalled >= 2: + logger.info("Feed reached the bottom after %d scrolls", step) + break container = _feed_container(driver) if container is not None: @@ -285,11 +436,18 @@ def main(argv: list[str] | None = None) -> bool: try: driver, browser = launch_driver(settings) logger.info("Using %s (%s)", browser.executable, browser.version) + _enable_response_logging(driver) logger.info("Validating Upwork login") login(driver, settings, lambda message: logger.warning(message)) + _log_document_responses(driver, logger) logger.info("Upwork login validation succeeded") return True + except UpworkBlockedError as exc: + _log_document_responses(driver, logger) + logger.error("%s", exc) + return False except Exception: + _log_document_responses(driver, logger) logger.exception("Login validation failed") return False finally: @@ -305,9 +463,11 @@ def main(argv: list[str] | None = None) -> bool: create_db(conn, cursor) driver, browser = launch_driver(settings) logger.info("Using %s (%s)", browser.executable, browser.version) + _enable_response_logging(driver) logger.info("Checking Upwork authentication") login(driver, settings, lambda message: logger.warning(message)) logger.info("Upwork authentication ready") + _log_document_responses(driver, logger) _load_job_list(driver, logger) @@ -320,6 +480,13 @@ def main(argv: list[str] | None = None) -> bool: job_posts = text.split("Posted")[1:] job_urls = _job_urls(driver) logger.info("Parsed %d job posts and found %d job links", len(job_posts), len(job_urls)) + if not job_urls: + # Reporting success here would hide both a block page and a changed + # feed markup behind "0 jobs processed". + raise RuntimeError( + "No job links were found on the Best Matches feed; the page markup may have " + "changed, or Upwork served an unexpected page" + ) counts = _process_jobs(job_posts, job_urls, cursor, logger) conn.commit() logger.info( @@ -333,7 +500,12 @@ def main(argv: list[str] | None = None) -> bool: counts.skipped, ) return True + except UpworkBlockedError as exc: + _log_document_responses(driver, logger) + logger.error("%s", exc) + return False except Exception: + _log_document_responses(driver, logger) logger.exception("Scraping failed") return False finally: diff --git a/upwork_scraper/auth.py b/upwork_scraper/auth.py index 3e9bd72..5aaab37 100644 --- a/upwork_scraper/auth.py +++ b/upwork_scraper/auth.py @@ -40,6 +40,58 @@ """ +BLOCKED_PAGE_JS = """ +const title = (document.title || '').toLowerCase(); +if (title.includes('attention required') || title.includes('access denied')) return true; +// Cloudflare's block page is a few kilobytes of plain HTML. Bail out on large +// documents first so checking a loaded feed does not pay for a layout pass. +const text = ((document.body && document.body.textContent) || '').toLowerCase(); +if (text.length > 50000) return false; +return text.includes('you have been blocked') + || text.includes('unable to access upwork.com') + || text.includes('why have i been blocked'); +""" +BLOCKED_MESSAGE = ( + "Cloudflare refused to pass {url} through to Upwork and returned its " + '"You have been blocked" page. The request never reached Upwork, so this is bot ' + "scoring rejecting this automated browser session, not a login or page problem. " + "Do not retry immediately: each attempt extends the block. Wait several minutes, " + "then run once. If Upwork loads normally in your own Chrome on this machine, the " + "network is fine and only the automation session is being penalised." +) + + +class UpworkBlockedError(RuntimeError): + """Raised when Cloudflare refuses to serve Upwork to the automated browser.""" + + +def _blocked_page(driver) -> bool: + """Report whether Upwork served Cloudflare's hard block page. + + Unlike an interstitial challenge, this page cannot clear itself: the request + never reached Upwork, so no widget, cookie, or reload will recover it. + """ + + try: + return bool(driver.execute_script(BLOCKED_PAGE_JS)) + except InvalidSessionIdException: + raise + except WebDriverException: + # A navigation can destroy the execution context mid-check. + return False + + +def ensure_not_blocked(driver) -> None: + """Fail fast with an actionable message instead of scraping a block page.""" + + if _blocked_page(driver): + try: + url = driver.current_url + except WebDriverException: + url = BEST_MATCHES_URL + raise UpworkBlockedError(BLOCKED_MESSAGE.format(url=url)) + + def _visible_button(driver, button_ids=()): for button_id in button_ids: button = _visible_element_by_id(driver, button_id) @@ -139,6 +191,7 @@ def _wait_for_login_field(driver, timeout: int, logger): deadline = time.monotonic() + timeout challenge_reported = False while time.monotonic() < deadline: + ensure_not_blocked(driver) if _authenticated(driver): return True if _cloudflare_challenge(driver): @@ -214,6 +267,10 @@ def _click_submit(driver, field, button_ids, timeout: int, logger) -> None: def _authenticated(driver) -> bool: try: + if _blocked_page(driver): + # The block page is not a login page, but treating it as an + # authenticated session makes the scraper silently scrape it. + return False parsed_url = urlsplit(driver.current_url) hostname = parsed_url.hostname or "" if hostname != "upwork.com" and not hostname.endswith(".upwork.com"): @@ -228,13 +285,18 @@ def _authenticated(driver) -> bool: def _at_best_matches(driver) -> bool: - parsed_url = urlsplit(driver.current_url) - return parsed_url.path.rstrip("/") == "/nx/find-work/best-matches" + # Upwork redirects the public path to its session-scoped counterpart + # (/nx/s/...), so both forms mean "already on Best Matches". + path = urlsplit(driver.current_url).path.rstrip("/") + return path in ("/nx/find-work/best-matches", "/nx/s/find-work/best-matches") def _go_to_best_matches(driver) -> None: if not _at_best_matches(driver): driver.get(BEST_MATCHES_URL) + # A 403 block page is served at the requested URL, so the address alone + # cannot tell a loaded feed from a refusal. + ensure_not_blocked(driver) def _visible_element_by_id(driver, element_id: str): @@ -270,8 +332,20 @@ def login(driver, settings: Settings, logger: Callable[[str], None]) -> None: _go_to_best_matches(driver) return - driver.get(LOGIN_URL) - login_step = _wait_for_login_field(driver, settings.verification_timeout, logger) + # Open the target page directly. A persistent profile is normally still + # signed in, and routing through the login page makes the browser follow a + # redirect and then start a second top-level navigation within a second, + # which Cloudflare scores as automation. An expired session simply + # redirects here to the login form, where the flow below continues. + driver.get(BEST_MATCHES_URL) + ensure_not_blocked(driver) + try: + login_step = _wait_for_login_field(driver, settings.verification_timeout, logger) + except TimeoutException: + # Neither the feed nor a login form came back; retry on the login page. + driver.get(LOGIN_URL) + login_step = _wait_for_login_field(driver, settings.verification_timeout, logger) + if login_step is True: _go_to_best_matches(driver) return @@ -328,6 +402,7 @@ def login(driver, settings: Settings, logger: Callable[[str], None]) -> None: reported_warning: str | None = None challenge_reported = False while time.monotonic() < deadline: + ensure_not_blocked(driver) if _authenticated(driver): _go_to_best_matches(driver) return diff --git a/upwork_scraper/browser.py b/upwork_scraper/browser.py index 3d58455..0f58c3f 100644 --- a/upwork_scraper/browser.py +++ b/upwork_scraper/browser.py @@ -81,6 +81,9 @@ def _launch_with_driver(settings: Settings, spec: BrowserSpec, driver_path: Path options.headless = False options.page_load_strategy = "eager" options.add_argument("--disable-dev-shm-usage") + # Lets the app report the HTTP outcome of each Upwork document load + # (status, cf-ray, cf-mitigated) so block pages can be diagnosed. + options.set_capability("goog:loggingPrefs", {"performance": "ALL"}) options.user_data_dir = str(settings.browser_profile_dir) if settings.proxy_server: options.add_argument(f"--proxy-server={settings.proxy_server}")