lancedb · jiaoew1991 · Sep 24, 2025
diff --git a/lance-spark-base_2.12/src/main/java/com/lancedb/lance/spark/utils/SchemaConverter.java b/lance-spark-base_2.12/src/main/java/com/lancedb/lance/spark/utils/SchemaConverter.java
@@ -115,7 +115,7 @@ private static StructType addVectorMetadata(
           // Validate element type is FloatType or DoubleType
           if (elementType instanceof FloatType || elementType instanceof DoubleType) {
             // Add metadata for FixedSizeList
-            Long vectorSize = Long.parseLong(properties.get(vectorSizeProperty));
+            long vectorSize = Long.parseLong(properties.get(vectorSizeProperty));
             Metadata newMetadata =
                 new MetadataBuilder()
                     .withMetadata(field.metadata())

diff --git a/lance-spark-base_2.12/src/main/java/org/apache/spark/sql/vectorized/LanceArrayAccessor.java b/lance-spark-base_2.12/src/main/java/org/apache/spark/sql/vectorized/LanceArrayAccessor.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.vectorized;
+
+import org.apache.arrow.vector.complex.ListVector;
+
+public class LanceArrayAccessor extends ArrowColumnVector.ArrowVectorAccessor {
+
+  private final ListVector accessor;
+  private final LanceArrowColumnVector arrayData;
+
+  public LanceArrayAccessor(ListVector vector) {
+    super(vector);
+    this.accessor = vector;
+    this.arrayData = new LanceArrowColumnVector(vector.getDataVector());
+  }
+
+  @Override
+  final ColumnarArray getArray(int rowId) {
+    int start = accessor.getElementStartIndex(rowId);
+    int end = accessor.getElementEndIndex(rowId);
+    return new ColumnarArray(arrayData, start, end - start);
+  }
+}
diff --git a/...spark-base_2.12/src/main/java/org/apache/spark/sql/vectorized/LanceArrowColumnVector.java b/...spark-base_2.12/src/main/java/org/apache/spark/sql/vectorized/LanceArrowColumnVector.java
@@ -16,13 +16,15 @@
 import org.apache.arrow.vector.UInt8Vector;
 import org.apache.arrow.vector.ValueVector;
 import org.apache.arrow.vector.complex.FixedSizeListVector;
+import org.apache.arrow.vector.complex.ListVector;
 import org.apache.spark.sql.types.Decimal;
 import org.apache.spark.sql.util.LanceArrowUtils;
 import org.apache.spark.unsafe.types.UTF8String;
 
 public class LanceArrowColumnVector extends ColumnVector {
   private UInt8Accessor uInt8Accessor;
   private FixedSizeListAccessor fixedSizeListAccessor;
+  private LanceArrayAccessor arrayAccessor;
   private ArrowColumnVector arrowColumnVector;
 
   public LanceArrowColumnVector(ValueVector vector) {
@@ -32,6 +34,8 @@ public LanceArrowColumnVector(ValueVector vector) {
     } else if (vector instanceof FixedSizeListVector) {
       // Handle FixedSizeListVector with custom accessor
       fixedSizeListAccessor = new FixedSizeListAccessor((FixedSizeListVector) vector);
+    } else if (vector instanceof ListVector) {
+      arrayAccessor = new LanceArrayAccessor((ListVector) vector);
     } else {
       arrowColumnVector = new ArrowColumnVector(vector);
     }
@@ -45,6 +49,9 @@ public void close() {
     if (fixedSizeListAccessor != null) {
       fixedSizeListAccessor.close();
     }
+    if (arrayAccessor != null) {
+      arrayAccessor.close();
+    }
     if (arrowColumnVector != null) {
       arrowColumnVector.close();
     }
@@ -58,6 +65,9 @@ public boolean hasNull() {
     if (fixedSizeListAccessor != null) {
       return fixedSizeListAccessor.getNullCount() > 0;
     }
+    if (arrayAccessor != null) {
+      return arrayAccessor.getNullCount() > 0;
+    }
     if (arrowColumnVector != null) {
       return arrowColumnVector.hasNull();
     }
@@ -72,6 +82,9 @@ public int numNulls() {
     if (fixedSizeListAccessor != null) {
       return fixedSizeListAccessor.getNullCount();
     }
+    if (arrayAccessor != null) {
+      return arrayAccessor.getNullCount();
+    }
     if (arrowColumnVector != null) {
       return arrowColumnVector.numNulls();
     }
@@ -86,6 +99,9 @@ public boolean isNullAt(int rowId) {
     if (fixedSizeListAccessor != null) {
       return fixedSizeListAccessor.isNullAt(rowId);
     }
+    if (arrayAccessor != null) {
+      return arrayAccessor.isNullAt(rowId);
+    }
     if (arrowColumnVector != null) {
       return arrowColumnVector.isNullAt(rowId);
     }
@@ -156,6 +172,9 @@ public ColumnarArray getArray(int rowId) {
     if (fixedSizeListAccessor != null) {
       return fixedSizeListAccessor.getArray(rowId);
     }
+    if (arrayAccessor != null) {
+      return arrayAccessor.getArray(rowId);
+    }
     if (arrowColumnVector != null) {
       return arrowColumnVector.getArray(rowId);
     }

diff --git a/lance-spark-base_2.12/src/main/scala/org/apache/spark/sql/util/LanceArrowUtils.scala b/lance-spark-base_2.12/src/main/scala/org/apache/spark/sql/util/LanceArrowUtils.scala
@@ -56,6 +56,15 @@ object LanceArrowUtils {
         val elementType = fromArrowField(elementField)
         val containsNull = elementField.isNullable
         ArrayType(elementType, containsNull)
+      case l: ArrowType.List =>
+        val children = field.getChildren
+        if (children.isEmpty) {
+          throw new SparkException(s"List field ${field.getName} has no children")
+        }
+        val elementField = children.get(0)
+        val elementType = fromArrowField(elementField)
+        val containsNull = elementField.isNullable
+        ArrayType(elementType, containsNull)
       case _ => ArrowUtils.fromArrowField(field)
     }
   }