Add sandbox filesystem commands and filer

acceptance/acceptance_test.go

@@ -190,6 +190,7 @@ func setReplsForTestEnvVars(t *testing.T, repls *testdiff.ReplacementsContext) {
 		"TEST_DEFAULT_WAREHOUSE_ID",
 		"TEST_DEFAULT_CLUSTER_ID",
 		"TEST_INSTANCE_POOL_ID",
+		"TEST_SANDBOX_ID",
 	}
 	for _, envVar := range envVars {
 		if value := os.Getenv(envVar); value != "" {
@@ -449,6 +450,9 @@ func testAccept(t *testing.T, inprocessMode bool, selectedTests []string, skipTo
 		if os.Getenv("TEST_INSTANCE_POOL_ID") == "" {
 			t.Setenv("TEST_INSTANCE_POOL_ID", testserver.TestDefaultInstancePoolId)
 		}
+		if os.Getenv("TEST_SANDBOX_ID") == "" {
+			t.Setenv("TEST_SANDBOX_ID", testserver.TestDefaultSandboxId)
+		}
 	}
 
 	const sharedProxyHint = "; re-run with -debugsandbox to see which test caused this"

acceptance/cmd/sandbox/fs/cat/hello.txt

@@ -0,0 +1 @@
+hello world!

acceptance/cmd/sandbox/fs/cat/out.test.toml

@@ -0,0 +1 @@
+Cloud = true

acceptance/cmd/sandbox/fs/cat/output.txt

@@ -0,0 +1,34 @@
+
+=== Help
+>>> [CLI] sandbox fs cat --help
+Show file content. Use SANDBOX:/absolute/path for sandbox paths.
+
+Usage:
+  databricks sandbox fs cat FILE_PATH [flags]
+
+Examples:
+  databricks sandbox fs cat my-sandbox:/tmp/file
+
+Flags:
+  -h, --help   help for cat
+
+Global Flags:
+      --debug            enable debug logging
+  -o, --output type      output type: text or json (default text)
+  -p, --profile string   ~/.databrickscfg profile
+  -t, --target string    bundle target to use (if applicable)
+
+=== cat a file
+>>> [CLI] sandbox fs cat [TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/hello.txt
+hello world!
+
+=== cat a directory, then a missing path
+>>> [CLI] sandbox fs cat [TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]
+Error: not a file: /tmp/sandbox-fs-[UNIQUE_NAME]
+
+Exit code: 1
+
+>>> [CLI] sandbox fs cat [TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/missing
+Error: file does not exist: /tmp/sandbox-fs-[UNIQUE_NAME]/missing
+
+Exit code: 1

acceptance/cmd/sandbox/fs/cat/script

@@ -0,0 +1,11 @@
+title "Help"
+trace $CLI sandbox fs cat --help
+
+$CLI sandbox fs cp hello.txt $SANDBOX_ROOT/hello.txt > /dev/null
+
+title "cat a file"
+trace $CLI sandbox fs cat $SANDBOX_ROOT/hello.txt
+
+title "cat a directory, then a missing path"
+errcode trace $CLI sandbox fs cat $SANDBOX_ROOT
+errcode trace $CLI sandbox fs cat $SANDBOX_ROOT/missing

acceptance/cmd/sandbox/fs/cp/local.txt

@@ -0,0 +1 @@
+hello world!

acceptance/cmd/sandbox/fs/cp/localdir/file1.txt

@@ -0,0 +1 @@
+file1 content

acceptance/cmd/sandbox/fs/cp/localdir/file2.txt

@@ -0,0 +1 @@
+file2 content

acceptance/cmd/sandbox/fs/cp/out.test.toml

@@ -0,0 +1 @@
+Cloud = true

acceptance/cmd/sandbox/fs/cp/output.txt

@@ -0,0 +1,77 @@
+
+=== Help
+>>> [CLI] sandbox fs cp --help
+Copy files and directories between your local filesystem and sandboxes.
+
+  Use SANDBOX:/absolute/path for sandbox paths.
+
+  Recursively copying a directory will copy all files inside directory
+  at SOURCE_PATH to the directory at TARGET_PATH.
+
+  When copying a file, if TARGET_PATH is a directory, the file will be created
+  inside the directory, otherwise the file is created at TARGET_PATH.
+
+  Files that already exist at the target are skipped unless --overwrite is set.
+
+Usage:
+  databricks sandbox fs cp SOURCE_PATH TARGET_PATH [flags]
+
+Examples:
+  databricks sandbox fs cp ./file my-sandbox:/tmp/file
+  databricks sandbox fs cp --overwrite ./file my-sandbox:/tmp/file
+  databricks sandbox fs cp -r my-sandbox:/tmp/dir ./local/dir
+
+Flags:
+      --concurrency int   number of parallel copy operations (default 8)
+  -h, --help              help for cp
+      --overwrite         overwrite existing files
+  -r, --recursive         recursively copy files from directory
+
+Global Flags:
+      --debug            enable debug logging
+  -o, --output type      output type: text or json (default text)
+  -p, --profile string   ~/.databrickscfg profile
+  -t, --target string    bundle target to use (if applicable)
+
+=== Upload a file
+>>> [CLI] sandbox fs cp local.txt [TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/file.txt
+local.txt -> [TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/file.txt
+
+>>> [CLI] sandbox fs cat [TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/file.txt
+hello world!
+
+=== Writing over an existing file needs --overwrite
+>>> [CLI] sandbox fs cp local.txt [TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/file.txt
+local.txt -> [TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/file.txt (skipped; already exists)
+
+>>> [CLI] sandbox fs cp --overwrite updated.txt [TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/file.txt
+updated.txt -> [TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/file.txt
+
+>>> [CLI] sandbox fs cat [TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/file.txt
+hello again!
+
+=== Copying a file to a directory puts it inside
+>>> [CLI] sandbox fs cp local.txt [TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/target
+local.txt -> [TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/target/local.txt
+
+>>> [CLI] sandbox fs ls [TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/target
+local.txt
+
+=== Writing into a directory that does not exist fails
+>>> [CLI] sandbox fs cp local.txt [TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/missing/file.txt
+Error: no such directory: /tmp/sandbox-fs-[UNIQUE_NAME]/missing
+
+Exit code: 1
+
+=== Copy a directory recursively, and back
+>>> [CLI] sandbox fs cp -r localdir [TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/dir
+localdir/file1.txt -> [TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/dir/file1.txt
+localdir/file2.txt -> [TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/dir/file2.txt
+
+>>> [CLI] sandbox fs cp -r [TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/dir downloaded
+[TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/dir/file1.txt -> downloaded/file1.txt
+[TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/dir/file2.txt -> downloaded/file2.txt
+
+>>> cat downloaded/file1.txt downloaded/file2.txt
+file1 content
+file2 content

acceptance/cmd/sandbox/fs/cp/script

@@ -0,0 +1,25 @@
+title "Help"
+trace $CLI sandbox fs cp --help
+
+title "Upload a file"
+trace $CLI sandbox fs cp local.txt $SANDBOX_ROOT/file.txt
+trace $CLI sandbox fs cat $SANDBOX_ROOT/file.txt
+
+title "Writing over an existing file needs --overwrite"
+trace $CLI sandbox fs cp local.txt $SANDBOX_ROOT/file.txt
+trace $CLI sandbox fs cp --overwrite updated.txt $SANDBOX_ROOT/file.txt
+trace $CLI sandbox fs cat $SANDBOX_ROOT/file.txt
+
+title "Copying a file to a directory puts it inside"
+$CLI sandbox fs mkdir $SANDBOX_ROOT/target
+trace $CLI sandbox fs cp local.txt $SANDBOX_ROOT/target
+trace $CLI sandbox fs ls $SANDBOX_ROOT/target
+
+title "Writing into a directory that does not exist fails"
+errcode trace $CLI sandbox fs cp local.txt $SANDBOX_ROOT/missing/file.txt
+
+title "Copy a directory recursively, and back"
+trace $CLI sandbox fs cp -r localdir $SANDBOX_ROOT/dir 2>&1 | sort
+trace $CLI sandbox fs cp -r $SANDBOX_ROOT/dir downloaded 2>&1 | sort
+trace cat downloaded/file1.txt downloaded/file2.txt
+rm -r downloaded

acceptance/cmd/sandbox/fs/cp/test.toml

@@ -0,0 +1 @@
+Ignore = ["downloaded"]

acceptance/cmd/sandbox/fs/cp/updated.txt

@@ -0,0 +1 @@
+hello again!

acceptance/cmd/sandbox/fs/ls/hello.txt

@@ -0,0 +1 @@
+hello world!

acceptance/cmd/sandbox/fs/ls/out.test.toml

@@ -0,0 +1 @@
+Cloud = true

acceptance/cmd/sandbox/fs/ls/output.txt

@@ -0,0 +1,41 @@
+
+=== Help
+>>> [CLI] sandbox fs ls --help
+Lists files. Use SANDBOX:/absolute/path for sandbox paths.
+
+Usage:
+  databricks sandbox fs ls DIR_PATH [flags]
+
+Examples:
+  databricks sandbox fs ls my-sandbox:/tmp
+
+Flags:
+      --absolute   Displays absolute paths.
+  -h, --help       help for ls
+  -l, --long       Displays full information including size, file type and modification time since Epoch in milliseconds.
+
+Global Flags:
+      --debug            enable debug logging
+  -o, --output type      output type: text or json (default text)
+  -p, --profile string   ~/.databrickscfg profile
+  -t, --target string    bundle target to use (if applicable)
+
+=== List a directory
+>>> [CLI] sandbox fs ls [TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]
+dir
+hello.txt
+
+>>> [CLI] sandbox fs ls --absolute [TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]
+[TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/dir
+[TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/hello.txt
+
+=== List a file, then a missing path
+>>> [CLI] sandbox fs ls [TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/hello.txt
+Error: not a directory: /tmp/sandbox-fs-[UNIQUE_NAME]/hello.txt
+
+Exit code: 1
+
+>>> [CLI] sandbox fs ls [TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/missing
+Error: no such directory: /tmp/sandbox-fs-[UNIQUE_NAME]/missing
+
+Exit code: 1

acceptance/cmd/sandbox/fs/ls/script

@@ -0,0 +1,13 @@
+title "Help"
+trace $CLI sandbox fs ls --help
+
+$CLI sandbox fs cp hello.txt $SANDBOX_ROOT/hello.txt > /dev/null
+$CLI sandbox fs mkdir $SANDBOX_ROOT/dir
+
+title "List a directory"
+trace $CLI sandbox fs ls $SANDBOX_ROOT
+trace $CLI sandbox fs ls --absolute $SANDBOX_ROOT
+
+title "List a file, then a missing path"
+errcode trace $CLI sandbox fs ls $SANDBOX_ROOT/hello.txt
+errcode trace $CLI sandbox fs ls $SANDBOX_ROOT/missing

acceptance/cmd/sandbox/fs/mkdir/hello.txt

@@ -0,0 +1 @@
+hello world!

acceptance/cmd/sandbox/fs/mkdir/out.test.toml

@@ -0,0 +1 @@
+Cloud = true

acceptance/cmd/sandbox/fs/mkdir/output.txt

@@ -0,0 +1,37 @@
+
+=== Help
+>>> [CLI] sandbox fs mkdir --help
+Make directories. Use SANDBOX:/absolute/path for sandbox paths.
+
+Usage:
+  databricks sandbox fs mkdir DIR_PATH [flags]
+
+Aliases:
+  mkdir, mkdirs
+
+Examples:
+  databricks sandbox fs mkdir my-sandbox:/tmp/a/b
+
+Flags:
+  -h, --help   help for mkdir
+
+Global Flags:
+      --debug            enable debug logging
+  -o, --output type      output type: text or json (default text)
+  -p, --profile string   ~/.databrickscfg profile
+  -t, --target string    bundle target to use (if applicable)
+
+=== Create nested directories
+>>> [CLI] sandbox fs mkdir [TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/a/b/c
+
+>>> [CLI] sandbox fs ls [TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/a/b
+c
+
+=== Creating an existing directory succeeds
+>>> [CLI] sandbox fs mkdir [TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/a/b/c
+
+=== Creating a directory over a file fails
+>>> [CLI] sandbox fs mkdir [TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/hello.txt
+Error: file already exists: /tmp/sandbox-fs-[UNIQUE_NAME]/hello.txt
+
+Exit code: 1

acceptance/cmd/sandbox/fs/mkdir/script

@@ -0,0 +1,13 @@
+title "Help"
+trace $CLI sandbox fs mkdir --help
+
+title "Create nested directories"
+trace $CLI sandbox fs mkdir $SANDBOX_ROOT/a/b/c
+trace $CLI sandbox fs ls $SANDBOX_ROOT/a/b
+
+title "Creating an existing directory succeeds"
+trace $CLI sandbox fs mkdir $SANDBOX_ROOT/a/b/c
+
+title "Creating a directory over a file fails"
+$CLI sandbox fs cp hello.txt $SANDBOX_ROOT/hello.txt > /dev/null
+errcode trace $CLI sandbox fs mkdir $SANDBOX_ROOT/hello.txt

acceptance/cmd/sandbox/fs/rm/hello.txt

@@ -0,0 +1 @@
+hello world!

acceptance/cmd/sandbox/fs/rm/out.test.toml

@@ -0,0 +1 @@
+Cloud = true

acceptance/cmd/sandbox/fs/rm/output.txt

@@ -0,0 +1,47 @@
+
+=== Help
+>>> [CLI] sandbox fs rm --help
+Remove files and directories. Use SANDBOX:/absolute/path for sandbox paths.
+
+Usage:
+  databricks sandbox fs rm PATH [flags]
+
+Examples:
+  databricks sandbox fs rm -r my-sandbox:/tmp/dir
+
+Flags:
+  -h, --help        help for rm
+  -r, --recursive   Recursively delete a non-empty directory.
+
+Global Flags:
+      --debug            enable debug logging
+  -o, --output type      output type: text or json (default text)
+  -p, --profile string   ~/.databrickscfg profile
+  -t, --target string    bundle target to use (if applicable)
+
+=== Remove a file
+>>> [CLI] sandbox fs rm [TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/hello.txt
+
+>>> [CLI] sandbox fs cat [TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/hello.txt
+Error: file does not exist: /tmp/sandbox-fs-[UNIQUE_NAME]/hello.txt
+
+Exit code: 1
+
+=== Removing a non-empty directory needs -r
+>>> [CLI] sandbox fs rm [TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/dir
+Error: directory not empty: /tmp/sandbox-fs-[UNIQUE_NAME]/dir
+
+Exit code: 1
+
+>>> [CLI] sandbox fs rm -r [TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/dir
+
+>>> [CLI] sandbox fs ls [TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/dir
+Error: no such directory: /tmp/sandbox-fs-[UNIQUE_NAME]/dir
+
+Exit code: 1
+
+=== Removing a missing path fails
+>>> [CLI] sandbox fs rm [TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/missing
+Error: file does not exist: /tmp/sandbox-fs-[UNIQUE_NAME]/missing
+
+Exit code: 1

acceptance/cmd/sandbox/fs/rm/script

@@ -0,0 +1,18 @@
+title "Help"
+trace $CLI sandbox fs rm --help
+
+$CLI sandbox fs cp hello.txt $SANDBOX_ROOT/hello.txt > /dev/null
+$CLI sandbox fs mkdir $SANDBOX_ROOT/dir
+$CLI sandbox fs cp hello.txt $SANDBOX_ROOT/dir/hello.txt > /dev/null
+
+title "Remove a file"
+trace $CLI sandbox fs rm $SANDBOX_ROOT/hello.txt
+errcode trace $CLI sandbox fs cat $SANDBOX_ROOT/hello.txt
+
+title "Removing a non-empty directory needs -r"
+errcode trace $CLI sandbox fs rm $SANDBOX_ROOT/dir
+trace $CLI sandbox fs rm -r $SANDBOX_ROOT/dir
+errcode trace $CLI sandbox fs ls $SANDBOX_ROOT/dir
+
+title "Removing a missing path fails"
+errcode trace $CLI sandbox fs rm $SANDBOX_ROOT/missing

acceptance/cmd/sandbox/fs/script.prepare

@@ -0,0 +1,14 @@
+# Cloud environments without a sandbox skip these tests. Locally, the harness sets
+# TEST_SANDBOX_ID to the testserver's default sandbox.
+if [ -z "${TEST_SANDBOX_ID:-}" ]; then
+  echo "SKIP_TEST TEST_SANDBOX_ID is not set"
+  exit 0
+fi
+
+# Each test works in its own directory under /tmp and removes it on exit.
+SANDBOX_ROOT="$TEST_SANDBOX_ID:/tmp/sandbox-fs-$UNIQUE_NAME"
+sandbox_fs_cleanup() {
+  $CLI sandbox fs rm -r "$SANDBOX_ROOT" 2>/dev/null || true
+}
+trap sandbox_fs_cleanup EXIT
+$CLI sandbox fs mkdir "$SANDBOX_ROOT"

acceptance/cmd/sandbox/fs/test.toml

@@ -0,0 +1,3 @@
+# Runs against the testserver fake of the sandbox file APIs locally, and against the existing
+# sandbox named by TEST_SANDBOX_ID on cloud, so a run doesn't create a sandbox per test.
+Cloud = true