Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@
import org.apache.iotdb.rpc.TSStatusCode;

import org.apache.thrift.TException;
import org.awaitility.core.ConditionTimeoutException;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
Expand All @@ -61,6 +62,7 @@
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
Expand Down Expand Up @@ -390,6 +392,9 @@ public void testPipeAfterDataRegionLeaderStop() {
}
return;
} catch (final Exception e) {
if (!isRetryablePipeFailure(e)) {
throw new AssertionError("Pipe replication failed due to a non-transient failure", e);
}
if (retry < 4) {
IoTDBTestLogger.logger.warn(
"Retrying table pipe leader-stop scenario after attempt {}", retry + 1, e);
Expand All @@ -416,10 +421,7 @@ private void insertDataAfterLeaderRestart(final String database) {
try {
TableModelUtils.insertData(database, database, 200, 300, senderEnv);
} catch (final RuntimeException e) {
if (e.getCause() instanceof SQLException sqlException
&& sqlException.getMessage() != null
&& (sqlException.getMessage().contains("Connection reset by peer")
|| sqlException.getMessage().contains("Connection refused"))) {
if (isTransientRpcFailure(e)) {
throw new AssertionError(
"DataNode internal RPC connections have not recovered after leader restart",
e);
Expand All @@ -429,6 +431,49 @@ private void insertDataAfterLeaderRestart(final String database) {
});
}

private static boolean isTransientRpcFailure(final Throwable failure) {
for (Throwable cause = failure; cause != null; cause = cause.getCause()) {
if (cause instanceof SQLException sqlException) {
for (SQLException nextException = sqlException;
nextException != null;
nextException = nextException.getNextException()) {
if (isTransientRpcFailureMessage(nextException.getMessage())) {
return true;
}
}
}

if (isTransientRpcFailureMessage(cause.getMessage())) {
return true;
}
}
return false;
}

private static boolean isRetryablePipeFailure(final Throwable failure) {
for (Throwable cause = failure; cause != null; cause = cause.getCause()) {
if (cause instanceof ConditionTimeoutException) {
return false;
}
}
return isTransientRpcFailure(failure);
}

private static boolean isTransientRpcFailureMessage(final String message) {
if (message == null) {
return false;
}

final String lowerCaseMessage = message.toLowerCase(Locale.ROOT);
return lowerCaseMessage.contains("connection reset")
|| lowerCaseMessage.contains("connection refused")
|| lowerCaseMessage.contains("broken pipe")
|| lowerCaseMessage.contains("socket is closed by peer")
|| lowerCaseMessage.contains("connection aborted")
|| lowerCaseMessage.contains("forcibly closed by the remote host")
|| lowerCaseMessage.contains("aborted by the software in your host machine");
}

private int restartTableDataRegionLeader(
final SyncConfigNodeIServiceClient client, final String database) throws TException {
final List<TRegionInfo> leaderRegionInfoList =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -602,6 +602,8 @@ public void testReceiverRestartWhenTransferring() throws Exception {
"select count(*) from root.db.**",
"count(root.db.d1.s1),",
Collections.singleton("1,"));
// Keep the verified baseline row durable across the receiver restart.
TestUtils.executeNonQuery(receiverEnv, "flush");

// Explicitly enqueue events while the receiver is offline instead of relying on a writer's
// sleep interval to overlap the restart. Failures propagate on the test thread.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,13 +29,15 @@
import org.apache.iotdb.itbase.env.BaseEnv;
import org.apache.iotdb.rpc.IoTDBConnectionException;
import org.apache.iotdb.rpc.StatementExecutionException;
import org.apache.iotdb.rpc.TSStatusCode;

import org.apache.tsfile.enums.ColumnCategory;
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.file.metadata.enums.CompressionType;
import org.apache.tsfile.file.metadata.enums.TSEncoding;
import org.apache.tsfile.read.common.TimeRange;
import org.apache.tsfile.write.record.Tablet;
import org.awaitility.Awaitility;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assert;
Expand All @@ -51,6 +53,7 @@
import java.sql.Statement;
import java.util.Arrays;
import java.util.Locale;
import java.util.concurrent.TimeUnit;

import static org.apache.iotdb.db.it.utils.TestUtils.createUser;
import static org.junit.Assert.assertEquals;
Expand Down Expand Up @@ -425,8 +428,8 @@ public void testInsert() throws IoTDBConnectionException, StatementExecutionExce
// grant an irrelevant privilege so that the new users can use database
sessionRoot.executeNonQueryStatement("GRANT SELECT ON DATABASE \"汉化\" TO USER userC");
sessionRoot.executeNonQueryStatement("GRANT SELECT ON DATABASE \"汉化\" TO USER userD");
sessionC.executeNonQueryStatement("USE \"汉化\"");
sessionD.executeNonQueryStatement("USE \"汉化\"");
awaitDatabaseAccess(sessionC, "USE \"汉化\"", "汉化");
awaitDatabaseAccess(sessionD, "USE \"汉化\"", "汉化");
// userC no privilege
try {
sessionC.executeNonQueryStatement(
Expand Down Expand Up @@ -867,8 +870,8 @@ public void testDelete() throws IoTDBConnectionException, StatementExecutionExce
// grant an irrelevant privilege so that the new users can use database
sessionRoot.executeNonQueryStatement("GRANT SELECT ON DATABASE test2 TO USER userC");
sessionRoot.executeNonQueryStatement("GRANT SELECT ON DATABASE test2 TO USER userD");
sessionC.executeNonQueryStatement("USE test2");
sessionD.executeNonQueryStatement("USE test2");
awaitDatabaseAccess(sessionC, "USE test2", "test2");
awaitDatabaseAccess(sessionD, "USE test2", "test2");
// userC no privilege
try {
sessionC.executeNonQueryStatement("DELETE FROM table1");
Expand Down Expand Up @@ -1155,4 +1158,25 @@ public void testAlter() throws IoTDBConnectionException, StatementExecutionExcep
}
}
}

private static void awaitDatabaseAccess(
final ITableSession session, final String useStatement, final String database)
throws IoTDBConnectionException, StatementExecutionException {
Awaitility.await()
.atMost(30, TimeUnit.SECONDS)
.pollInterval(100, TimeUnit.MILLISECONDS)
.untilAsserted(
() -> {
try {
session.executeNonQueryStatement(useStatement);
} catch (StatementExecutionException e) {
if (e.getStatusCode() == TSStatusCode.NO_PERMISSION.getStatusCode()
&& e.getMessage() != null
&& e.getMessage().contains("Access Denied: DATABASE " + database)) {
throw new AssertionError("Database access has not propagated for " + database, e);
}
throw e;
}
});
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -40,13 +40,15 @@
import org.apache.iotdb.pipe.api.exception.PipeConnectionException;
import org.apache.iotdb.pipe.api.exception.PipeException;

import com.google.common.util.concurrent.Uninterruptibles;
import org.junit.Assert;
import org.junit.Test;
import org.mockito.Mockito;

import java.util.concurrent.CountDownLatch;
import java.util.concurrent.FutureTask;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;

import static org.awaitility.Awaitility.await;
Expand Down Expand Up @@ -108,6 +110,7 @@ public void testExternalSinkDiscardEventsOfPipeNotBlockedByConnectionRetry() thr

final PipeSinkSubtask subtask =
new PipeSinkSubtask(
null,
"PipeSinkSubtaskTest",
System.currentTimeMillis(),
"data_test",
Expand Down Expand Up @@ -258,6 +261,7 @@ public void testExternalSinkCloseNotConcurrentWithConnectionRetry() throws Excep

final PipeSinkSubtask subtask =
new PipeSinkSubtask(
null,
"PipeSinkSubtaskTest",
System.currentTimeMillis(),
"data_test",
Expand All @@ -269,12 +273,11 @@ public void testExternalSinkCloseNotConcurrentWithConnectionRetry() throws Excep
final Thread failureThread =
new Thread(() -> subtask.onFailure(new PipeConnectionException("connection broken")));
failureThread.setDaemon(true);
final FutureTask<Long> closeTask =
final FutureTask<Void> closeTask =
new FutureTask<>(
() -> {
final long startTime = System.nanoTime();
subtask.close();
return TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTime);
return null;
});
final Thread closeThread = new Thread(closeTask);
closeThread.setDaemon(true);
Expand All @@ -287,7 +290,7 @@ public void testExternalSinkCloseNotConcurrentWithConnectionRetry() throws Excep
closeThread.start();
closeStarted = true;

Assert.assertTrue(closeTask.get(30, TimeUnit.SECONDS) < 1000);
closeTask.get(1, TimeUnit.SECONDS);
Assert.assertFalse(closeEntered.await(100, TimeUnit.MILLISECONDS));
Assert.assertFalse(closeDuringHandshake.get());
} finally {
Expand All @@ -313,17 +316,18 @@ public void testExternalSinkCloseNotConcurrentWithConnectionRetry() throws Excep
public void testExternalSinkCloseDoesNotWaitForeverForConnectorClose() throws Exception {
final int originalTimeout =
CommonDescriptor.getInstance().getConfig().getDnConnectionTimeoutInMS();
CommonDescriptor.getInstance().getConfig().setDnConnectionTimeoutInMS(30);

final PipeConnector connector = mock(PipeConnector.class);
final UnboundedBlockingPendingQueue<?> pendingQueue = mock(UnboundedBlockingPendingQueue.class);
final CountDownLatch closeEntered = new CountDownLatch(1);
final CountDownLatch releaseClose = new CountDownLatch(1);
final CountDownLatch connectorCloseReturned = new CountDownLatch(1);

doAnswer(
invocation -> {
closeEntered.countDown();
releaseClose.await(2, TimeUnit.SECONDS);
Uninterruptibles.awaitUninterruptibly(releaseClose);
connectorCloseReturned.countDown();
return null;
})
.when(connector)
Expand All @@ -339,25 +343,38 @@ public void testExternalSinkCloseDoesNotWaitForeverForConnectorClose() throws Ex
(UnboundedBlockingPendingQueue) pendingQueue,
connector);

try {
final long startTime = System.currentTimeMillis();
subtask.close();
final long closeDuration = System.currentTimeMillis() - startTime;
final FutureTask<Void> closeTask =
new FutureTask<>(
() -> {
subtask.close();
return null;
});
final Thread closeThread = new Thread(closeTask);
closeThread.setDaemon(true);

Assert.assertTrue(closeDuration < 1000);
try {
CommonDescriptor.getInstance().getConfig().setDnConnectionTimeoutInMS(30);
closeThread.start();
Assert.assertTrue(closeEntered.await(30, TimeUnit.SECONDS));
closeTask.get(1, TimeUnit.SECONDS);
Assert.assertEquals(1L, connectorCloseReturned.getCount());
} finally {
releaseClose.countDown();
CommonDescriptor.getInstance().getConfig().setDnConnectionTimeoutInMS(originalTimeout);
try {
closeThread.join(30_000);
} finally {
CommonDescriptor.getInstance().getConfig().setDnConnectionTimeoutInMS(originalTimeout);
}
}

Assert.assertFalse(closeThread.isAlive());
Assert.assertTrue(connectorCloseReturned.await(30, TimeUnit.SECONDS));
}

@Test
public void testBuiltinSinkCloseWaitsForConnectorClose() throws Exception {
final int originalTimeout =
CommonDescriptor.getInstance().getConfig().getDnConnectionTimeoutInMS();
CommonDescriptor.getInstance().getConfig().setDnConnectionTimeoutInMS(30);

final PipeConnector connector = mock(PipeConnector.class);
final UnboundedBlockingPendingQueue<?> pendingQueue = mock(UnboundedBlockingPendingQueue.class);
final CountDownLatch closeEntered = new CountDownLatch(1);
Expand All @@ -383,22 +400,20 @@ public void testBuiltinSinkCloseWaitsForConnectorClose() throws Exception {
(UnboundedBlockingPendingQueue) pendingQueue,
connector,
false);
final CountDownLatch closeReturned = new CountDownLatch(1);
final Thread closeThread =
new Thread(
final FutureTask<Void> closeTask =
new FutureTask<>(
() -> {
try {
subtask.close();
} finally {
closeReturned.countDown();
}
subtask.close();
return null;
});
final Thread closeThread = new Thread(closeTask);
closeThread.setDaemon(true);

try {
CommonDescriptor.getInstance().getConfig().setDnConnectionTimeoutInMS(30);
closeThread.start();
Assert.assertTrue(closeEntered.await(30, TimeUnit.SECONDS));
Assert.assertFalse(closeReturned.await(100, TimeUnit.MILLISECONDS));
Assert.assertThrows(TimeoutException.class, () -> closeTask.get(100, TimeUnit.MILLISECONDS));
} finally {
releaseClose.countDown();
try {
Expand All @@ -409,7 +424,7 @@ public void testBuiltinSinkCloseWaitsForConnectorClose() throws Exception {
}

Assert.assertFalse(closeThread.isAlive());
Assert.assertEquals(0L, closeReturned.getCount());
closeTask.get();
}

@Test
Expand Down
Loading