diff --git a/CHANGELOG.md b/CHANGELOG.md index d3f3dc8b..957853dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Release History +## Unreleased + +- Report untyped NULL columns as STRING in kernel result metadata, matching Thrift. (#470) + ## 3.0.0 - Upgrade the native kernel backend to 1.4.0. diff --git a/lib/kernel/KernelArrowIpc.ts b/lib/kernel/KernelArrowIpc.ts index 63dc2873..c0dda5bd 100644 --- a/lib/kernel/KernelArrowIpc.ts +++ b/lib/kernel/KernelArrowIpc.ts @@ -191,7 +191,8 @@ function arrowTypeToTTypeId(field: Field): TTypeId { return TTypeId.STRUCT_TYPE; case 'NULL': case 'VOID': - return TTypeId.NULL_TYPE; + // Match Thrift's STRING metadata for untyped NULL columns. + return TTypeId.STRING_TYPE; default: break; } @@ -248,7 +249,6 @@ function arrowTypeToTTypeId(field: Field): TTypeId { if (DataType.isList(arrowType)) return TTypeId.ARRAY_TYPE; if (DataType.isMap(arrowType)) return TTypeId.MAP_TYPE; if (DataType.isStruct(arrowType)) return TTypeId.STRUCT_TYPE; - if (DataType.isNull(arrowType)) return TTypeId.NULL_TYPE; return TTypeId.STRING_TYPE; } diff --git a/tests/unit/kernel/KernelOperationBackend.test.ts b/tests/unit/kernel/KernelOperationBackend.test.ts index d11d5c5e..972de4fc 100644 --- a/tests/unit/kernel/KernelOperationBackend.test.ts +++ b/tests/unit/kernel/KernelOperationBackend.test.ts @@ -27,6 +27,7 @@ import { Float32, Float64, Utf8, + Null, Binary, DateDay, TimestampMicrosecond, @@ -37,6 +38,7 @@ import { } from 'apache-arrow'; import KernelOperationBackend from '../../../lib/kernel/KernelOperationBackend'; +import { TTypeId } from '../../../thrift/TCLIService_types'; import ClientContextStub from '../.stubs/ClientContextStub'; // Minimal stub of the napi `Statement` surface that emits a precomputed @@ -226,6 +228,37 @@ describe('KernelOperationBackend — M0 datatype round-trip via napi → ArrowRe expect(await backend.hasMore()).to.equal(false); }); + // The kernel normalizes VOID to Utf8 but preserves the original type metadata. + for (const nullField of [ + withTypeName(new Field('untyped_null', new Utf8(), true), 'NULL'), + withTypeName(new Field('untyped_null', new Utf8(), true), 'VOID'), + new Field('untyped_null', new Null(), true), + ]) { + const typeName = nullField.metadata.get('databricks.type_name') ?? 'Arrow Null without metadata'; + it(`reports untyped ${typeName} as STRING alongside a typed NULL STRING without changing values`, async () => { + const schema = new Schema([nullField, withTypeName(new Field('typed_null_string', new Utf8(), true), 'STRING')]); + const backend = new KernelOperationBackend({ + statement: new StatementStub(ipcSchemaOnly(schema), [ + ipcFromColumns(schema, { untyped_null: [null], typed_null_string: [null] }), + ]), + context: new ClientContextStub(), + }); + + try { + const metadata = await backend.getResultMetadata(); + expect(metadata.schema?.columns.map((column) => column.typeDesc.types[0].primitiveEntry?.type)).to.deep.equal([ + TTypeId.STRING_TYPE, + TTypeId.STRING_TYPE, + ]); + expect(await backend.fetchChunk({ limit: 100 })).to.deep.equal([ + { untyped_null: null, typed_null_string: null }, + ]); + } finally { + await backend.close(); + } + }); + } + it('preserves decimal qualifiers for a schema-only result with zero scale', async () => { const schema = new Schema([new Field('amount', new Decimal(0, 38, 128), true)]); const backend = new KernelOperationBackend({