From 6faa9f9cb87d13773c70264edc6029a3c1e4a59f Mon Sep 17 00:00:00 2001 From: Dmitry Meyer Date: Mon, 26 Oct 2026 10:08:59 +0000 Subject: [PATCH] Fix Ctrl+C handling in CLI confirm_ask turned Ctrl+C into sys.exit(1), so handlers that catch a second Ctrl+C at a prompt never ran: `dstack apply` didn't abort the run, and preset creation left the agent and its runs going. Handle KeyboardInterrupt once in main(): print "Interrupted" and re-raise with the traceback hidden. On an uncaught KeyboardInterrupt, Python shuts down normally and then kills itself with SIGINT, so a calling shell script stops too. Exiting with 130 doesn't do that: bash assumes the child handled SIGINT itself and continues. - Drop the interrupt handlers in confirm_ask, login, _repo_diff_verbose (library code), and the apply and preset catch-alls, which exited 0. - Re-raise after aborting a run on a second Ctrl+C. - Let Ctrl+C through questionary menus (unsafe_ask). - Respect -y when interrupting fleet provisioning on old servers. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/dstack/_internal/cli/commands/apply.py | 59 +++++++++--------- src/dstack/_internal/cli/commands/login.py | 18 ++---- src/dstack/_internal/cli/commands/preset.py | 6 +- src/dstack/_internal/cli/commands/project.py | 5 +- src/dstack/_internal/cli/main.py | 14 +++++ .../cli/services/configurators/fleet.py | 2 +- .../cli/services/configurators/run.py | 1 + src/dstack/_internal/cli/utils/common.py | 6 +- .../_internal/core/models/repos/remote.py | 21 +++---- .../cli/services/configurators/test_run.py | 38 +++++++++++- .../cli/services/presets/test_create.py | 32 ++++++++++ src/tests/_internal/cli/test_main.py | 60 +++++++++++++++++++ 12 files changed, 191 insertions(+), 71 deletions(-) create mode 100644 src/tests/_internal/cli/test_main.py diff --git a/src/dstack/_internal/cli/commands/apply.py b/src/dstack/_internal/cli/commands/apply.py index 7e37522a8e..bc627ac441 100644 --- a/src/dstack/_internal/cli/commands/apply.py +++ b/src/dstack/_internal/cli/commands/apply.py @@ -1,6 +1,5 @@ import argparse import shlex -import sys from argcomplete import FilesCompleter # type: ignore[attr-defined] @@ -71,37 +70,33 @@ def _register(self): ) def _command(self, args: argparse.Namespace): - try: - if args.help is not NOTSET: - if args.help is not None: - configurator_class = get_apply_configurator_class( - ApplyConfigurationType(args.help) - ) - configurator_class.register_args(self._parser) - self._parser.print_help() - return - self._parser.print_help() - console.print( - "\nType `dstack apply -h CONFIGURATION_TYPE` to see configuration-specific options.\n" + if args.help is not NOTSET: + if args.help is not None: + configurator_class = get_apply_configurator_class( + ApplyConfigurationType(args.help) ) + configurator_class.register_args(self._parser) + self._parser.print_help() return - - super()._command(args) - if not args.yes and args.configuration_file == APPLY_STDIN_NAME: - raise CLIError("Cannot read configuration from stdin if -y/--yes is not specified") - configuration_path, configuration = load_apply_configuration(args.configuration_file) - configurator_class = get_apply_configurator_class(configuration.type) - configurator = configurator_class(api_client=self.api) - configurator_parser = configurator.get_parser() - configurator_args, unknown_args = configurator_parser.parse_known_args(args.extra_args) - if unknown_args: - raise CLIError(f"Unrecognized arguments: {shlex.join(unknown_args)}") - configurator.apply_configuration( - conf=configuration, - configuration_path=configuration_path, - command_args=args, - configurator_args=configurator_args, + self._parser.print_help() + console.print( + "\nType `dstack apply -h CONFIGURATION_TYPE` to see configuration-specific options.\n" ) - except KeyboardInterrupt: - console.print("\nOperation interrupted by user. Exiting...") - sys.exit(0) + return + + super()._command(args) + if not args.yes and args.configuration_file == APPLY_STDIN_NAME: + raise CLIError("Cannot read configuration from stdin if -y/--yes is not specified") + configuration_path, configuration = load_apply_configuration(args.configuration_file) + configurator_class = get_apply_configurator_class(configuration.type) + configurator = configurator_class(api_client=self.api) + configurator_parser = configurator.get_parser() + configurator_args, unknown_args = configurator_parser.parse_known_args(args.extra_args) + if unknown_args: + raise CLIError(f"Unrecognized arguments: {shlex.join(unknown_args)}") + configurator.apply_configuration( + conf=configuration, + configuration_path=configuration_path, + command_args=args, + configurator_args=configurator_args, + ) diff --git a/src/dstack/_internal/cli/commands/login.py b/src/dstack/_internal/cli/commands/login.py index c7b7463e12..6ebfbdbec0 100644 --- a/src/dstack/_internal/cli/commands/login.py +++ b/src/dstack/_internal/cli/commands/login.py @@ -113,15 +113,11 @@ def _select_provider_or_error(self, api_client: APIClient, provider: Optional[st return provider def _prompt_url(self) -> str: - try: - url = UrlPrompt.ask( - "Enter the server URL", - default="https://sky.dstack.ai", - console=console, - ) - except KeyboardInterrupt: - console.print("\nCancelled by user") - sys.exit(1) + url = UrlPrompt.ask( + "Enter the server URL", + default="https://sky.dstack.ai", + console=console, + ) if url is None: raise CLIError("URL is required") return url @@ -135,9 +131,7 @@ def _prompt_provider(self, available_providers: list[str]) -> str: choices=choices, qmark="", instruction="(↑↓ Enter)", - ).ask() - if selected_provider is None: - sys.exit(1) + ).unsafe_ask() return selected_provider def _configure_projects( diff --git a/src/dstack/_internal/cli/commands/preset.py b/src/dstack/_internal/cli/commands/preset.py index ea557aaba9..28ba5e75ef 100644 --- a/src/dstack/_internal/cli/commands/preset.py +++ b/src/dstack/_internal/cli/commands/preset.py @@ -234,11 +234,7 @@ def _register(self) -> None: def _command(self, args: argparse.Namespace) -> None: super()._command(args) - try: - args.subfunc(args) - except KeyboardInterrupt: - console.print("\nOperation interrupted by user. Exiting...") - sys.exit(0) + args.subfunc(args) def _reconcile(self) -> None: """Finalize any detached/orphaned session whose agent already completed, diff --git a/src/dstack/_internal/cli/commands/project.py b/src/dstack/_internal/cli/commands/project.py index 867bd9686d..45a0ef6ef2 100644 --- a/src/dstack/_internal/cli/commands/project.py +++ b/src/dstack/_internal/cli/commands/project.py @@ -33,10 +33,11 @@ def select_default_project( default_project: Currently default project, if any. Returns: - Selected project configuration, or None if cancelled. + Selected project configuration. Raises: CLIError: If `is_project_menu_supported` is False or `project_configs` is empty. + KeyboardInterrupt: If the selection is cancelled. """ if not is_project_menu_supported: raise CLIError("Interactive menu is not supported on this platform") @@ -61,7 +62,7 @@ def select_default_project( default=default_value, # pyright: ignore[reportArgumentType] qmark="", instruction="(↑↓ Enter)", - ).ask() + ).unsafe_ask() if selected_index is not None and isinstance(selected_index, int): return project_configs[selected_index] diff --git a/src/dstack/_internal/cli/main.py b/src/dstack/_internal/cli/main.py index 84ae1e2863..5a78e4ce5d 100644 --- a/src/dstack/_internal/cli/main.py +++ b/src/dstack/_internal/cli/main.py @@ -102,6 +102,20 @@ def main(): error_console.print(f"[error]{escape(str(e))}[/]") logger.debug(e, exc_info=True) sys.exit(1) + except KeyboardInterrupt: + error_console.print("\nInterrupted") + logger.debug("Interrupted", exc_info=True) + # Re-raise instead of exiting with an error code: on an uncaught KeyboardInterrupt, + # Python shuts down normally (incl. atexit) and then kills itself with SIGINT, + # so that a parent shell sees the interrupt and stops, e.g., a script loop. + sys.excepthook = _excepthook_without_keyboard_interrupt + raise + + +def _excepthook_without_keyboard_interrupt(exc_type, exc_value, exc_traceback): + if issubclass(exc_type, KeyboardInterrupt): + return + sys.__excepthook__(exc_type, exc_value, exc_traceback) if __name__ == "__main__": diff --git a/src/dstack/_internal/cli/services/configurators/fleet.py b/src/dstack/_internal/cli/services/configurators/fleet.py index 12ef066a48..ce0db5192f 100644 --- a/src/dstack/_internal/cli/services/configurators/fleet.py +++ b/src/dstack/_internal/cli/services/configurators/fleet.py @@ -264,7 +264,7 @@ def _apply_plan_on_old_server(self, plan: FleetPlan, command_args: argparse.Name time.sleep(LIVE_TABLE_PROVISION_INTERVAL_SECS) fleet = self.api.client.fleets.get(self.api.project, fleet.name) except KeyboardInterrupt: - if confirm_ask("Delete the fleet before exiting?"): + if not command_args.yes and confirm_ask("Delete the fleet before exiting?"): with console.status("Deleting fleet..."): self.api.client.fleets.delete( project_name=self.api.project, names=[fleet.name] diff --git a/src/dstack/_internal/cli/services/configurators/run.py b/src/dstack/_internal/cli/services/configurators/run.py index 756c8396fa..568c38fab4 100644 --- a/src/dstack/_internal/cli/services/configurators/run.py +++ b/src/dstack/_internal/cli/services/configurators/run.py @@ -333,6 +333,7 @@ def apply_plan( console.print("Stopped") except KeyboardInterrupt: abort_at_exit = True + raise finally: run.detach() if abort_at_exit: diff --git a/src/dstack/_internal/cli/utils/common.py b/src/dstack/_internal/cli/utils/common.py index 71a76c846d..8f54978bc5 100644 --- a/src/dstack/_internal/cli/utils/common.py +++ b/src/dstack/_internal/cli/utils/common.py @@ -139,11 +139,7 @@ def configure_logging(): def confirm_ask(prompt, **kwargs) -> bool: kwargs["console"] = console - try: - return Confirm.ask(prompt=prompt, **kwargs) - except KeyboardInterrupt: - console.print("\nCancelled by user") - sys.exit(1) + return Confirm.ask(prompt=prompt, **kwargs) def add_row_from_dict(table: Table, data: dict[str, Any], **kwargs): diff --git a/src/dstack/_internal/core/models/repos/remote.py b/src/dstack/_internal/core/models/repos/remote.py index f51eb85e07..b047d147a5 100644 --- a/src/dstack/_internal/core/models/repos/remote.py +++ b/src/dstack/_internal/core/models/repos/remote.py @@ -1,7 +1,6 @@ import io import re import subprocess -import sys import time from dataclasses import dataclass from typing import Annotated, BinaryIO, Callable, Dict, Optional, Union, cast @@ -374,15 +373,11 @@ def _interactive_git_proc( def _repo_diff_verbose(repo: git.Repo, repo_hash: str, warning_time: float = 5) -> bytes: collector = _DiffCollector(warning_time) - try: - _interactive_git_proc(repo.git.diff(repo_hash, binary=True, as_process=True), collector) - for filename in repo.untracked_files: - _interactive_git_proc( - repo.git.diff("/dev/null", filename, no_index=True, binary=True, as_process=True), - collector, - ignore_status=True, - ) - return collector.get() - except KeyboardInterrupt: - print("\nAborted") - sys.exit(1) + _interactive_git_proc(repo.git.diff(repo_hash, binary=True, as_process=True), collector) + for filename in repo.untracked_files: + _interactive_git_proc( + repo.git.diff("/dev/null", filename, no_index=True, binary=True, as_process=True), + collector, + ignore_status=True, + ) + return collector.get() diff --git a/src/tests/_internal/cli/services/configurators/test_run.py b/src/tests/_internal/cli/services/configurators/test_run.py index a382ab2991..fbe80fddfb 100644 --- a/src/tests/_internal/cli/services/configurators/test_run.py +++ b/src/tests/_internal/cli/services/configurators/test_run.py @@ -2,14 +2,17 @@ from pathlib import Path from textwrap import dedent from typing import List, Optional, Tuple -from unittest.mock import Mock +from unittest.mock import MagicMock, Mock import pytest from gpuhunt import KNOWN_TENSTORRENT_ACCELERATORS +from rich.prompt import Confirm +import dstack._internal.cli.services.configurators.run as run_configurator_module from dstack._internal.cli.services.configurators import get_run_configurator_class from dstack._internal.cli.services.configurators.run import ( ServiceConfigurator, + TaskConfigurator, render_run_spec_diff, ) from dstack._internal.core.errors import CLIError, ConfigurationError @@ -24,6 +27,7 @@ ) from dstack._internal.core.models.envs import Env from dstack._internal.core.models.profiles import Profile +from dstack._internal.core.models.runs import RunStatus from dstack._internal.server.testing.common import get_run_spec from tests._internal.utils.test_ssh import PRIVATE_KEY, PUBLIC_KEY, PUBLIC_KEY_NO_COMMENT @@ -192,6 +196,38 @@ def test_composes_get_plan_and_apply_plan(self, monkeypatch): ) +class TestApplyPlan: + def test_interrupt_at_stop_prompt_aborts_run(self, monkeypatch: pytest.MonkeyPatch): + run = MagicMock(status=RunStatus.RUNNING) + run.attach.side_effect = KeyboardInterrupt + api = MagicMock() + api.runs.apply_plan.return_value = run + for name in [ + "print_run_plan", + "_print_service_urls", + "_print_dev_environment_connection_info", + ]: + monkeypatch.setattr(run_configurator_module, name, Mock()) + monkeypatch.setattr(run_configurator_module, "_get_apply_status", Mock(return_value="")) + monkeypatch.setattr(run_configurator_module, "get_runs_table", Mock(return_value="")) + monkeypatch.setattr( + run_configurator_module, "_is_ready_to_attach", Mock(return_value=True) + ) + # Submit the run, then press Ctrl+C at "Stop the run before detaching?" + monkeypatch.setattr(Confirm, "ask", Mock(side_effect=[True, KeyboardInterrupt])) + + with pytest.raises(KeyboardInterrupt): + TaskConfigurator(api_client=api).apply_plan( + run_plan=MagicMock(current_resource=None), + repo=Mock(), + command_args=argparse.Namespace(yes=False, detach=False, verbose=False), + configurator_args=argparse.Namespace(max_offers=None), + ssh_identity_file=None, + ) + + run.stop.assert_called_once_with(abort=True) + + class TestRenderRunSpecDiff: def test_diff(self): old = get_run_spec( diff --git a/src/tests/_internal/cli/services/presets/test_create.py b/src/tests/_internal/cli/services/presets/test_create.py index 3912d9cd94..959aeedd6d 100644 --- a/src/tests/_internal/cli/services/presets/test_create.py +++ b/src/tests/_internal/cli/services/presets/test_create.py @@ -3,9 +3,11 @@ import os import uuid from types import SimpleNamespace +from unittest.mock import Mock import pytest from pydantic import ValidationError +from rich.prompt import Confirm from dstack._internal.cli.models.preset_agent import ( PresetAgentInfo, @@ -30,6 +32,7 @@ _print_fleet_offers, _save_final_report_copy, _stop_active_session_runs, + _stop_or_detach_agent_session, _suspend_agent_session, create_preset, follow_preset, @@ -965,6 +968,35 @@ async def create(**kwargs): assert "dstack preset resume" in output assert sessions[0].name in output + def test_interrupt_at_stop_prompt_stops_agent(self, tmp_path, monkeypatch): + session_dir = tmp_path / "ab12cd34" + session_dir.mkdir() + session = PresetSession(path=session_dir, preset_id="ab12cd34") + # A live agent: this very process. + session.write_state( + get_session_state( + run=get_session_run( + session_process=PresetSessionProcess(pid=os.getpid(), started_at=None) + ) + ) + ) + stopped = [] + monkeypatch.setattr( + "dstack._internal.cli.services.presets.create.terminate_agent_process", + lambda process: stopped.append("terminate"), + ) + monkeypatch.setattr( + "dstack._internal.cli.services.presets.create._stop_active_session_runs", + lambda api, session: stopped.append("stop_runs"), + ) + monkeypatch.setattr(Confirm, "ask", Mock(side_effect=KeyboardInterrupt)) + + _stop_or_detach_agent_session(session, SimpleNamespace()) + + assert stopped == ["terminate", "stop_runs"] + state = json.loads((session_dir / "session.json").read_text()) + assert state["status"] == "interrupted" + def test_suspend_scrubs_workspace_token(self, tmp_path, capsys): session_dir = tmp_path / "ab12cd34" session_dir.mkdir() diff --git a/src/tests/_internal/cli/test_main.py b/src/tests/_internal/cli/test_main.py new file mode 100644 index 0000000000..7a5b42dbd7 --- /dev/null +++ b/src/tests/_internal/cli/test_main.py @@ -0,0 +1,60 @@ +import signal +import subprocess +import sys +from textwrap import dedent +from unittest.mock import Mock + +import pytest + +import dstack._internal.cli.main as main_module +from dstack._internal.compat import IS_WINDOWS +from tests._internal.cli.common import run_dstack_cli + + +class TestMain: + def test_reraises_keyboard_interrupt(self, monkeypatch: pytest.MonkeyPatch, capsys): + monkeypatch.setattr(sys, "excepthook", sys.excepthook) + monkeypatch.setattr(main_module, "check_for_updates", Mock(side_effect=KeyboardInterrupt)) + + with pytest.raises(KeyboardInterrupt): + run_dstack_cli([]) + + assert "Interrupted" in capsys.readouterr().err + assert sys.excepthook is main_module._excepthook_without_keyboard_interrupt + + @pytest.mark.skipif(IS_WINDOWS, reason="Windows has no exit-by-signal status") + def test_process_exits_via_sigint_without_traceback(self): + # Spawns a Python process: only the interpreter's own handling of an uncaught exception + # shows that the CLI exits via SIGINT and that the excepthook hides the traceback. + code = dedent( + """ + import sys + from unittest.mock import patch + + sys.argv = ["dstack"] + with patch("dstack._internal.cli.main.check_for_updates", side_effect=KeyboardInterrupt): + from dstack._internal.cli.main import main + + main() + """ + ) + + result = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True) + + assert result.returncode == -signal.SIGINT + assert "Interrupted" in result.stderr + assert "Traceback" not in result.stderr + + +class TestExcepthookWithoutKeyboardInterrupt: + def test_hides_keyboard_interrupt(self, capsys): + main_module._excepthook_without_keyboard_interrupt( + KeyboardInterrupt, KeyboardInterrupt(), None + ) + + assert capsys.readouterr().err == "" + + def test_prints_other_exceptions(self, capsys): + main_module._excepthook_without_keyboard_interrupt(ValueError, ValueError("boom"), None) + + assert "ValueError: boom" in capsys.readouterr().err