Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 24 additions & 11 deletions paimon-api/src/main/java/org/apache/paimon/CoreOptions.java
Original file line number Diff line number Diff line change
Expand Up @@ -5134,20 +5134,33 @@ public boolean withVectorFormat() {
return options.contains(VECTOR_FILE_FORMAT);
}

public Set<String> vectorField() {
String vectorFields = options.get(CoreOptions.VECTOR_FIELD);
if (vectorFields == null || vectorFields.trim().isEmpty()) {
return Collections.emptySet();
}
return Arrays.stream(vectorFields.trim().split(",")).collect(Collectors.toSet());
/** Resolves {@link #VECTOR_FIELD} against the given field names. */
public Set<String> vectorField(List<String> fieldNames) {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P2] Keep the published vector-field overloads callable

Paimon 2.0.0 exposes both CoreOptions.vectorField() and CoreOptions.vectorField(Map). I compiled an unchanged Java caller against the pre-change API. It printed [embedding] for both calls on that build, then failed with NoSuchMethodError at the first call when loaded with this PR's API. Trimming in the built-in catalogs does not require removing either entry point. Keep both overloads with their legacy parsing semantics, deprecate them if desired, and use the schema-aware overloads inside Paimon.

return resolveVectorFields(options.get(VECTOR_FIELD), fieldNames);
}

/** Resolves {@link #VECTOR_FIELD} against the given field names. */
public static Set<String> vectorField(Map<String, String> options, List<String> fieldNames) {
return resolveVectorFields(options.get(VECTOR_FIELD.key()), fieldNames);
}

public static Set<String> vectorField(Map<String, String> options) {
String vectorFields = options.getOrDefault(CoreOptions.VECTOR_FIELD.key(), null);
if (vectorFields == null || vectorFields.trim().isEmpty()) {
return Collections.emptySet();
/**
* Uses a name exactly as written when a field has that name, which keeps existing tables with
* spaces in field names working, and its trimmed form otherwise.
*/
private static Set<String> resolveVectorFields(
@Nullable String vectorFields, List<String> fieldNames) {
Set<String> fields = new HashSet<>();
if (vectorFields == null) {
return fields;
}
for (String field : vectorFields.trim().split(",")) {
String name = fieldNames.contains(field) ? field : field.trim();
if (!name.isEmpty()) {
fields.add(name);
}
}
return Arrays.stream(vectorFields.trim().split(",")).collect(Collectors.toSet());
return fields;
}

public long vectorTargetFileSize() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -390,7 +390,7 @@ public static void validateTableSchema(TableSchema schema, Set<String> dynamicOp
}

// vector field names must point to vector type
Set<String> fieldNamesSpecifiedAsVector = options.vectorField();
Set<String> fieldNamesSpecifiedAsVector = options.vectorField(schema.fieldNames());
schema.fields()
.forEach(
f ->
Expand Down
16 changes: 16 additions & 0 deletions paimon-core/src/test/java/org/apache/paimon/CoreOptionsTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,9 @@

import org.junit.jupiter.api.Test;

import java.util.Arrays;
import java.util.List;

import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;

Expand Down Expand Up @@ -175,6 +178,19 @@ public void testChangelogRowDeduplicateIgnoreFieldsTrim() {
assertThat(options.changelogRowDeduplicateIgnoreFields()).containsExactly("f1", "f2", "f3");
}

@Test
public void testVectorFieldTrim() {
Options conf = new Options();
conf.set(CoreOptions.VECTOR_FIELD, " v1 , v2, v3 ");
// An exact field name such as " v3" wins over its trimmed form.
List<String> fieldNames = Arrays.asList("v1", "v2", " v3");

assertThat(new CoreOptions(conf).vectorField(fieldNames))
.containsExactlyInAnyOrder("v1", "v2", " v3");
assertThat(CoreOptions.vectorField(conf.toMap(), fieldNames))
.containsExactlyInAnyOrder("v1", "v2", " v3");
}

@Test
public void testIgnoreGlobalIndexColumnUpdateAction() {
Options conf = new Options();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,23 @@ public void testVectorRowIdsAfterNormalOnlyFiles(String format) throws Exception
.containsExactly(null, null, 22F, 33F);
}

@Test
public void testVectorFieldNameWithLeadingSpace() throws Exception {
// Existing tables may name a vector field " embedding_v2" and list it as "embedding,
// embedding_v2", so the exact name must win over its trimmed form.
Schema schema =
vectorSchema("json")
.column(" embedding_v2", DataTypes.VECTOR(2, DataTypes.FLOAT()))
.option(CoreOptions.VECTOR_FIELD.key(), "embedding, embedding_v2")
.build();
catalog.createTable(identifier(), schema, false);
write(getTableDefault(), GenericRow.of(0, vector(1), vector(2)));

List<InternalRow> rows = read(getTableDefault());
assertThat(rows).hasSize(1);
assertThat(rows.get(0).getVector(2).toFloatArray()[0]).isEqualTo(2F);
}

@Test
public void testPartialUpdateOfSharedVectorFile() throws Exception {
Schema schema =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1565,6 +1565,35 @@ public void testVectorStoreUnknownColumn() {
.hasMessage("Some of the columns specified as vector-field are unknown.");
}

@Test
public void testVectorFieldWithSpaces() {
Map<String, String> options = new HashMap<>();
options.put(BUCKET.key(), String.valueOf(-1));
options.put(CoreOptions.ROW_TRACKING_ENABLED.key(), "true");
options.put(DATA_EVOLUTION_ENABLED.key(), "true");
options.put(CoreOptions.FILE_FORMAT.key(), "avro");
options.put(VECTOR_FILE_FORMAT.key(), "json");
options.put(VECTOR_FIELD.key(), "v1, v2");

List<DataField> fields =
Arrays.asList(
new DataField(0, "f0", DataTypes.INT()),
new DataField(1, "v1", DataTypes.VECTOR(6, DataTypes.FLOAT())),
new DataField(2, "v2", DataTypes.VECTOR(6, DataTypes.FLOAT())));
assertThatCode(
() ->
validateTableSchema(
new TableSchema(
1,
fields,
10,
emptyList(),
emptyList(),
options,
"")))
.doesNotThrowAnyException();
}

@Test
public void testVectorStoreContainsNonVectorColumn() {
Map<String, String> options = new HashMap<>();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1320,6 +1320,7 @@ public static Schema fromCatalogTable(CatalogBaseTable catalogTable) {
.orElse(Collections.emptyList()))
.partitionKeys(getPartitionKeys(catalogTable));
Map<String, String> columnComments = getColumnComments(catalogTable);
Set<String> vectorFields = CoreOptions.vectorField(options, rowType.getFieldNames());
rowType.getFields()
.forEach(
field ->
Expand All @@ -1330,7 +1331,8 @@ public static Schema fromCatalogTable(CatalogBaseTable catalogTable) {
field.getType(),
options,
blobFileFields,
blobInlineFields),
blobInlineFields,
vectorFields),
columnComments.get(field.getName()),
null,
generationExpressions.get(field.getName())));
Expand All @@ -1350,14 +1352,14 @@ private static org.apache.paimon.types.DataType resolveDataType(
org.apache.flink.table.types.logical.LogicalType logicalType,
Map<String, String> options,
Set<String> blobFileFields,
Set<String> blobInlineFields) {
Set<String> blobInlineFields,
Set<String> vectorFields) {
if (blobInlineFields.contains(fieldName)) {
return toBlobType(logicalType, false);
}
if (blobFileFields.contains(fieldName)) {
return toBlobType(logicalType, true);
}
Set<String> vectorFields = CoreOptions.vectorField(options);
if (vectorFields.contains(fieldName)) {
return toVectorType(fieldName, logicalType, options);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -687,7 +687,6 @@ private Schema toInitialSchema(
List<String> blobFields = CoreOptions.blobField(properties);
Set<String> blobDescriptorFields = new CoreOptions(properties).blobDescriptorField();
List<String> blobViewFields = CoreOptions.blobViewField(properties);
Set<String> vectorFields = CoreOptions.fromMap(properties).vectorField();
String provider = properties.get(TableCatalog.PROP_PROVIDER);
if (!usePaimon(provider)) {
if (isFormatTable(provider)) {
Expand Down Expand Up @@ -731,6 +730,7 @@ private Schema toInitialSchema(
normalizedProperties.put(
CoreOptions.BUCKET_KEY.key(), resolveColumnNameList(bucketKey, fieldNames));
}
Set<String> vectorFields = CoreOptions.vectorField(properties, fieldNames);

Schema.Builder schemaBuilder =
Schema.newBuilder()
Expand Down
Loading