From e051b228d0340b76e95bfe906ff1296f91394636 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Fri, 6 Nov 2026 22:44:40 +0000 Subject: [PATCH 1/3] fix(kernel): report untyped NULL columns as STRING Signed-off-by: Vu Anh Phung --- CHANGELOG.md | 4 +++ lib/kernel/KernelArrowIpc.ts | 3 +- .../kernel/KernelOperationBackend.test.ts | 30 +++++++++++++++++++ 3 files changed, 36 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d3f3dc8b..957853dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Release History +## Unreleased + +- Report untyped NULL columns as STRING in kernel result metadata, matching Thrift. (#470) + ## 3.0.0 - Upgrade the native kernel backend to 1.4.0. diff --git a/lib/kernel/KernelArrowIpc.ts b/lib/kernel/KernelArrowIpc.ts index 63dc2873..f3f23a20 100644 --- a/lib/kernel/KernelArrowIpc.ts +++ b/lib/kernel/KernelArrowIpc.ts @@ -191,7 +191,8 @@ function arrowTypeToTTypeId(field: Field): TTypeId { return TTypeId.STRUCT_TYPE; case 'NULL': case 'VOID': - return TTypeId.NULL_TYPE; + // Match Thrift's STRING metadata for untyped NULL columns. + return TTypeId.STRING_TYPE; default: break; } diff --git a/tests/unit/kernel/KernelOperationBackend.test.ts b/tests/unit/kernel/KernelOperationBackend.test.ts index d11d5c5e..53a874b5 100644 --- a/tests/unit/kernel/KernelOperationBackend.test.ts +++ b/tests/unit/kernel/KernelOperationBackend.test.ts @@ -37,6 +37,7 @@ import { } from 'apache-arrow'; import KernelOperationBackend from '../../../lib/kernel/KernelOperationBackend'; +import { TTypeId } from '../../../thrift/TCLIService_types'; import ClientContextStub from '../.stubs/ClientContextStub'; // Minimal stub of the napi `Statement` surface that emits a precomputed @@ -226,6 +227,35 @@ describe('KernelOperationBackend — M0 datatype round-trip via napi → ArrowRe expect(await backend.hasMore()).to.equal(false); }); + for (const typeName of ['NULL', 'VOID']) { + it(`reports untyped ${typeName} as STRING alongside a typed NULL STRING without changing values`, async () => { + // The kernel normalizes VOID to Utf8 but preserves the original type metadata. + const schema = new Schema([ + withTypeName(new Field('untyped_null', new Utf8(), true), typeName), + withTypeName(new Field('typed_null_string', new Utf8(), true), 'STRING'), + ]); + const backend = new KernelOperationBackend({ + statement: new StatementStub(ipcSchemaOnly(schema), [ + ipcFromColumns(schema, { untyped_null: [null], typed_null_string: [null] }), + ]), + context: new ClientContextStub(), + }); + + try { + const metadata = await backend.getResultMetadata(); + expect(metadata.schema?.columns.map((column) => column.typeDesc.types[0].primitiveEntry?.type)).to.deep.equal([ + TTypeId.STRING_TYPE, + TTypeId.STRING_TYPE, + ]); + expect(await backend.fetchChunk({ limit: 100 })).to.deep.equal([ + { untyped_null: null, typed_null_string: null }, + ]); + } finally { + await backend.close(); + } + }); + } + it('preserves decimal qualifiers for a schema-only result with zero scale', async () => { const schema = new Schema([new Field('amount', new Decimal(0, 38, 128), true)]); const backend = new KernelOperationBackend({ From e41949e8eb75613f562d3606d37faa230a421319 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Fri, 6 Nov 2026 22:50:23 +0000 Subject: [PATCH 2/3] fix(kernel): normalize native Arrow Null metadata Signed-off-by: Vu Anh Phung --- lib/kernel/KernelArrowIpc.ts | 2 +- tests/unit/kernel/KernelOperationBackend.test.ts | 15 +++++++++------ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/lib/kernel/KernelArrowIpc.ts b/lib/kernel/KernelArrowIpc.ts index f3f23a20..aed16bc2 100644 --- a/lib/kernel/KernelArrowIpc.ts +++ b/lib/kernel/KernelArrowIpc.ts @@ -249,7 +249,7 @@ function arrowTypeToTTypeId(field: Field): TTypeId { if (DataType.isList(arrowType)) return TTypeId.ARRAY_TYPE; if (DataType.isMap(arrowType)) return TTypeId.MAP_TYPE; if (DataType.isStruct(arrowType)) return TTypeId.STRUCT_TYPE; - if (DataType.isNull(arrowType)) return TTypeId.NULL_TYPE; + if (DataType.isNull(arrowType)) return TTypeId.STRING_TYPE; return TTypeId.STRING_TYPE; } diff --git a/tests/unit/kernel/KernelOperationBackend.test.ts b/tests/unit/kernel/KernelOperationBackend.test.ts index 53a874b5..972de4fc 100644 --- a/tests/unit/kernel/KernelOperationBackend.test.ts +++ b/tests/unit/kernel/KernelOperationBackend.test.ts @@ -27,6 +27,7 @@ import { Float32, Float64, Utf8, + Null, Binary, DateDay, TimestampMicrosecond, @@ -227,13 +228,15 @@ describe('KernelOperationBackend — M0 datatype round-trip via napi → ArrowRe expect(await backend.hasMore()).to.equal(false); }); - for (const typeName of ['NULL', 'VOID']) { + // The kernel normalizes VOID to Utf8 but preserves the original type metadata. + for (const nullField of [ + withTypeName(new Field('untyped_null', new Utf8(), true), 'NULL'), + withTypeName(new Field('untyped_null', new Utf8(), true), 'VOID'), + new Field('untyped_null', new Null(), true), + ]) { + const typeName = nullField.metadata.get('databricks.type_name') ?? 'Arrow Null without metadata'; it(`reports untyped ${typeName} as STRING alongside a typed NULL STRING without changing values`, async () => { - // The kernel normalizes VOID to Utf8 but preserves the original type metadata. - const schema = new Schema([ - withTypeName(new Field('untyped_null', new Utf8(), true), typeName), - withTypeName(new Field('typed_null_string', new Utf8(), true), 'STRING'), - ]); + const schema = new Schema([nullField, withTypeName(new Field('typed_null_string', new Utf8(), true), 'STRING')]); const backend = new KernelOperationBackend({ statement: new StatementStub(ipcSchemaOnly(schema), [ ipcFromColumns(schema, { untyped_null: [null], typed_null_string: [null] }), From d9af9b9f75ce5a6aaaf7aef4e6a856f24dfa4753 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Fri, 6 Nov 2026 22:53:08 +0000 Subject: [PATCH 3/3] refactor(kernel): use STRING fallback for Arrow Null Signed-off-by: Vu Anh Phung --- lib/kernel/KernelArrowIpc.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/kernel/KernelArrowIpc.ts b/lib/kernel/KernelArrowIpc.ts index aed16bc2..c0dda5bd 100644 --- a/lib/kernel/KernelArrowIpc.ts +++ b/lib/kernel/KernelArrowIpc.ts @@ -249,7 +249,6 @@ function arrowTypeToTTypeId(field: Field): TTypeId { if (DataType.isList(arrowType)) return TTypeId.ARRAY_TYPE; if (DataType.isMap(arrowType)) return TTypeId.MAP_TYPE; if (DataType.isStruct(arrowType)) return TTypeId.STRUCT_TYPE; - if (DataType.isNull(arrowType)) return TTypeId.STRING_TYPE; return TTypeId.STRING_TYPE; }