Skip to content
This repository was archived by the owner on Jun 8, 2026. It is now read-only.
This repository was archived by the owner on Jun 8, 2026. It is now read-only.

tests.system.test_system_dbapi.TestTransactionsManagement: test_rollback failed #469

Closed

Description

@flaky-bot

This test failed!

To configure my behavior, see the Flaky Bot documentation.

If I'm commenting on this issue too often, add the flakybot: quiet label and
I will stop commenting.


commit: 2487800
buildURL: Build Status, Sponge
status: failed

Test output
args = (parent: "projects/precise-truck-742"
instance_id: "google-cloud-1627550679627"
instance {
  name: "projects/precise-t...1627550946"
  }
  labels {
    key: "python-spanner-dbapi-systests"
    value: "true"
  }
  processing_units: 1000
}
,)
kwargs = {'metadata': [('google-cloud-resource-prefix', 'projects/precise-truck-742/instances/google-cloud-1627550679627'), ('x...ms', 'parent=projects/precise-truck-742'), ('x-goog-api-client', 'gl-python/3.8.6 grpc/1.39.0 gax/1.31.1 gccl/3.6.0')]}
@six.wraps(callable_)
def error_remapped_callable(*args, **kwargs):
    try:
      return callable_(*args, **kwargs)

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/grpc_helpers.py:67:


self = <grpc._channel._UnaryUnaryMultiCallable object at 0x7fe7942014f0>
request = parent: "projects/precise-truck-742"
instance_id: "google-cloud-1627550679627"
instance {
name: "projects/precise-tr... "1627550946"
}
labels {
key: "python-spanner-dbapi-systests"
value: "true"
}
processing_units: 1000
}

timeout = None
metadata = [('google-cloud-resource-prefix', 'projects/precise-truck-742/instances/google-cloud-1627550679627'), ('x-goog-request...ams', 'parent=projects/precise-truck-742'), ('x-goog-api-client', 'gl-python/3.8.6 grpc/1.39.0 gax/1.31.1 gccl/3.6.0')]
credentials = None, wait_for_ready = None, compression = None

def __call__(self,
             request,
             timeout=None,
             metadata=None,
             credentials=None,
             wait_for_ready=None,
             compression=None):
    state, call, = self._blocking(request, timeout, metadata, credentials,
                                  wait_for_ready, compression)
  return _end_unary_response_blocking(state, call, False, None)

.nox/system-3-8/lib/python3.8/site-packages/grpc/_channel.py:946:


state = <grpc._channel._RPCState object at 0x7fe78e5b66d0>
call = <grpc._cython.cygrpc.SegregatedCall object at 0x7fe79445bc00>
with_call = False, deadline = None

def _end_unary_response_blocking(state, call, with_call, deadline):
    if state.code is grpc.StatusCode.OK:
        if with_call:
            rendezvous = _MultiThreadedRendezvous(state, call, None, deadline)
            return state.response, rendezvous
        else:
            return state.response
    else:
      raise _InactiveRpcError(state)

E grpc._channel._InactiveRpcError: <_InactiveRpcError of RPC that terminated with:
E status = StatusCode.ALREADY_EXISTS
E details = "Instance already exists: projects/precise-truck-742/instances/google-cloud-1627550679627"
E debug_error_string = "{"created":"@1627550946.587652143","description":"Error received from peer ipv4:74.125.195.95:443","file":"src/core/lib/surface/call.cc","file_line":1069,"grpc_message":"Instance already exists: projects/precise-truck-742/instances/google-cloud-1627550679627","grpc_status":6}"
E >

.nox/system-3-8/lib/python3.8/site-packages/grpc/_channel.py:849: _InactiveRpcError

The above exception was the direct cause of the following exception:

def setUpModule():
    if USE_EMULATOR:
        from google.auth.credentials import AnonymousCredentials

        emulator_project = os.getenv("GCLOUD_PROJECT", "emulator-test-project")
        Config.CLIENT = Client(
            project=emulator_project, credentials=AnonymousCredentials()
        )
    else:
        Config.CLIENT = Client()
    retry = RetryErrors(exceptions.ServiceUnavailable)

    configs = list(retry(Config.CLIENT.list_instance_configs)())

    instances = retry(_list_instances)()
    EXISTING_INSTANCES[:] = instances

    # Delete test instances that are older than an hour.
    cutoff = int(time.time()) - 1 * 60 * 60
    for instance_pb in Config.CLIENT.list_instances(
        "labels.python-spanner-dbapi-systests:true"
    ):
        instance = Instance.from_pb(instance_pb, Config.CLIENT)
        if "created" not in instance.labels:
            continue
        create_time = int(instance.labels["created"])
        if create_time > cutoff:
            continue
        # Instance cannot be deleted while backups exist.
        for backup_pb in instance.list_backups():
            backup = Backup.from_pb(backup_pb, instance)
            backup.delete()
        instance.delete()

    if CREATE_INSTANCE:
        if not USE_EMULATOR:
            # Defend against back-end returning configs for regions we aren't
            # actually allowed to use.
            configs = [config for config in configs if "-us-" in config.name]

        if not configs:
            raise ValueError("List instance configs failed in module set up.")

        Config.INSTANCE_CONFIG = configs[0]
        config_name = configs[0].name
        create_time = str(int(time.time()))
        labels = {"python-spanner-dbapi-systests": "true", "created": create_time}

        Config.INSTANCE = Config.CLIENT.instance(
            INSTANCE_ID, config_name, labels=labels
        )
      created_op = Config.INSTANCE.create()

tests/system/test_system_dbapi.py:98:


google/cloud/spanner_v1/instance.py:318: in create
future = api.create_instance(
google/cloud/spanner_admin_instance_v1/services/instance_admin/client.py:829: in create_instance
response = rpc(request, retry=retry, timeout=timeout, metadata=metadata,)
.nox/system-3-8/lib/python3.8/site-packages/google/api_core/gapic_v1/method.py:145: in call
return wrapped_func(*args, **kwargs)
.nox/system-3-8/lib/python3.8/site-packages/google/api_core/grpc_helpers.py:69: in error_remapped_callable
six.raise_from(exceptions.from_grpc_error(exc), exc)


value = None
from_value = <_InactiveRpcError of RPC that terminated with:
status = StatusCode.ALREADY_EXISTS
details = "Instance already exist...message":"Instance already exists: projects/precise-truck-742/instances/google-cloud-1627550679627","grpc_status":6}"

???
E google.api_core.exceptions.AlreadyExists: 409 Instance already exists: projects/precise-truck-742/instances/google-cloud-1627550679627

:3: AlreadyExists

Activity

  1. added
    flakybot: issueAn issue filed by the Flaky Bot. Should not be added manually.
    priority: p1Important issue which blocks shipping the next release. Will be fixed prior to next release.
    type: bugError or flaw in code with unintended results or allowing sub-optimal usage patterns.
    on Jul 29, 2021
  2. added
    type: processA process-related concern. May include testing, release, or the like.
    and removed
    priority: p1Important issue which blocks shipping the next release. Will be fixed prior to next release.
    type: bugError or flaw in code with unintended results or allowing sub-optimal usage patterns.
    on Jul 29, 2021
  3. flaky-bot commented on Jul 29, 2021

    @flaky-bot
    Author

    Test passed for commit 93c09ab (Build Status, Sponge)! Closing this issue.

  4. added
    flakybot: flakyTells the Flaky Bot not to close or comment on this issue.
    priority: p1Important issue which blocks shipping the next release. Will be fixed prior to next release.
    on Jul 30, 2021
  5. flaky-bot commented on Jul 30, 2021

    @flaky-bot
    Author

    Looks like this issue is flaky. 😟

    I'm going to leave this open and stop commenting.

    A human should fix and close this.


    commit: 06b4215
    buildURL: Build Status, Sponge
    status: failed

    Test output
    target = functools.partial(>)
    predicate = .if_exception_type_predicate at 0x7f151e5f25e0>
    sleep_generator = 
    deadline = 60, on_error = None
    
    def retry_target(target, predicate, sleep_generator, deadline, on_error=None):
        """Call a function and retry if it fails.
    
        This is the lowest-level retry helper. Generally, you'll use the
        higher-level retry helper :class:`Retry`.
    
        Args:
            target(Callable): The function to call and retry. This must be a
                nullary function - apply arguments with `functools.partial`.
            predicate (Callable[Exception]): A callable used to determine if an
                exception raised by the target should be considered retryable.
                It should return True to retry or False otherwise.
            sleep_generator (Iterable[float]): An infinite iterator that determines
                how long to sleep between retries.
            deadline (float): How long to keep retrying the target. The last sleep
                period is shortened as necessary, so that the last retry runs at
                ``deadline`` (and not considerably beyond it).
            on_error (Callable[Exception]): A function to call while processing a
                retryable exception.  Any error raised by this function will *not*
                be caught.
    
        Returns:
            Any: the return value of the target function.
    
        Raises:
            google.api_core.RetryError: If the deadline is exceeded while retrying.
            ValueError: If the sleep generator stops yielding values.
            Exception: If the target raises a method that isn't retryable.
        """
        if deadline is not None:
            deadline_datetime = datetime_helpers.utcnow() + datetime.timedelta(
                seconds=deadline
            )
        else:
            deadline_datetime = None
    
        last_exc = None
    
        for sleep in sleep_generator:
            try:
    
              return target()
    

    .nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:189:


    self = <google.api_core.operation.Operation object at 0x7f151c7e5e20>
    retry = <google.api_core.retry.Retry object at 0x7f151e5f6190>

    def _done_or_raise(self, retry=DEFAULT_RETRY):
        """Check if the future is done and raise if it's not."""
        kwargs = {} if retry is DEFAULT_RETRY else {"retry": retry}
    
        if not self.done(**kwargs):
    
          raise _OperationNotComplete()
    

    E google.api_core.future.polling._OperationNotComplete

    .nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:87: _OperationNotComplete

    The above exception was the direct cause of the following exception:

    self = <google.api_core.operation.Operation object at 0x7f151c7e5e20>
    timeout = 60, retry = <google.api_core.retry.Retry object at 0x7f151e5f6190>

    def _blocking_poll(self, timeout=None, retry=DEFAULT_RETRY):
        """Poll and wait for the Future to be resolved.
    
        Args:
            timeout (int):
                How long (in seconds) to wait for the operation to complete.
                If None, wait indefinitely.
        """
        if self._result_set:
            return
    
        retry_ = self._retry.with_deadline(timeout)
    
        try:
            kwargs = {} if retry is DEFAULT_RETRY else {"retry": retry}
    
          retry_(self._done_or_raise)(**kwargs)
    

    .nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:108:


    args = (), kwargs = {}
    target = functools.partial(<bound method PollingFuture._done_or_raise of <google.api_core.operation.Operation object at 0x7f151c7e5e20>>)
    sleep_generator = <generator object exponential_sleep_generator at 0x7f151c6bf0b0>

    @general_helpers.wraps(func)
    def retry_wrapped_func(*args, **kwargs):
        """A wrapper that calls target function with retry."""
        target = functools.partial(func, *args, **kwargs)
        sleep_generator = exponential_sleep_generator(
            self._initial, self._maximum, multiplier=self._multiplier
        )
    
      return retry_target(
    
            target,
            self._predicate,
            sleep_generator,
            self._deadline,
            on_error=on_error,
        )
    

    .nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:286:


    target = functools.partial(<bound method PollingFuture._done_or_raise of <google.api_core.operation.Operation object at 0x7f151c7e5e20>>)
    predicate = <function if_exception_type..if_exception_type_predicate at 0x7f151e5f25e0>
    sleep_generator = <generator object exponential_sleep_generator at 0x7f151c6bf0b0>
    deadline = 60, on_error = None

    def retry_target(target, predicate, sleep_generator, deadline, on_error=None):
        """Call a function and retry if it fails.
    
        This is the lowest-level retry helper. Generally, you'll use the
        higher-level retry helper :class:`Retry`.
    
        Args:
            target(Callable): The function to call and retry. This must be a
                nullary function - apply arguments with `functools.partial`.
            predicate (Callable[Exception]): A callable used to determine if an
                exception raised by the target should be considered retryable.
                It should return True to retry or False otherwise.
            sleep_generator (Iterable[float]): An infinite iterator that determines
                how long to sleep between retries.
            deadline (float): How long to keep retrying the target. The last sleep
                period is shortened as necessary, so that the last retry runs at
                ``deadline`` (and not considerably beyond it).
            on_error (Callable[Exception]): A function to call while processing a
                retryable exception.  Any error raised by this function will *not*
                be caught.
    
        Returns:
            Any: the return value of the target function.
    
        Raises:
            google.api_core.RetryError: If the deadline is exceeded while retrying.
            ValueError: If the sleep generator stops yielding values.
            Exception: If the target raises a method that isn't retryable.
        """
        if deadline is not None:
            deadline_datetime = datetime_helpers.utcnow() + datetime.timedelta(
                seconds=deadline
            )
        else:
            deadline_datetime = None
    
        last_exc = None
    
        for sleep in sleep_generator:
            try:
                return target()
    
            # pylint: disable=broad-except
            # This function explicitly must deal with broad exceptions.
            except Exception as exc:
                if not predicate(exc):
                    raise
                last_exc = exc
                if on_error is not None:
                    on_error(exc)
    
            now = datetime_helpers.utcnow()
    
            if deadline_datetime is not None:
                if deadline_datetime <= now:
    
                  six.raise_from(
    
                        exceptions.RetryError(
                            "Deadline of {:.1f}s exceeded while calling {}".format(
                                deadline, target
                            ),
                            last_exc,
                        ),
                        last_exc,
                    )
    

    .nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:204:


    value = None, from_value = _OperationNotComplete()

    ???
    E google.api_core.exceptions.RetryError: Deadline of 60.0s exceeded while calling functools.partial(<bound method PollingFuture._done_or_raise of <google.api_core.operation.Operation object at 0x7f151c7e5e20>>), last exception:

    :3: RetryError

    During handling of the above exception, another exception occurred:

    def setUpModule():
        if USE_EMULATOR:
            from google.auth.credentials import AnonymousCredentials
    
            emulator_project = os.getenv("GCLOUD_PROJECT", "emulator-test-project")
            Config.CLIENT = Client(
                project=emulator_project, credentials=AnonymousCredentials()
            )
        else:
            Config.CLIENT = Client()
        retry = RetryErrors(exceptions.ServiceUnavailable)
    
        configs = list(retry(Config.CLIENT.list_instance_configs)())
    
        instances = retry(_list_instances)()
        EXISTING_INSTANCES[:] = instances
    
        # Delete test instances that are older than an hour.
        cutoff = int(time.time()) - 1 * 60 * 60
        for instance_pb in Config.CLIENT.list_instances(
            "labels.python-spanner-dbapi-systests:true"
        ):
            instance = Instance.from_pb(instance_pb, Config.CLIENT)
            if "created" not in instance.labels:
                continue
            create_time = int(instance.labels["created"])
            if create_time > cutoff:
                continue
            # Instance cannot be deleted while backups exist.
            for backup_pb in instance.list_backups():
                backup = Backup.from_pb(backup_pb, instance)
                backup.delete()
            instance.delete()
    
        if CREATE_INSTANCE:
            if not USE_EMULATOR:
                # Defend against back-end returning configs for regions we aren't
                # actually allowed to use.
                configs = [config for config in configs if "-us-" in config.name]
    
            if not configs:
                raise ValueError("List instance configs failed in module set up.")
    
            Config.INSTANCE_CONFIG = configs[0]
            config_name = configs[0].name
            create_time = str(int(time.time()))
            labels = {"python-spanner-dbapi-systests": "true", "created": create_time}
    
            Config.INSTANCE = Config.CLIENT.instance(
                INSTANCE_ID, config_name, labels=labels
            )
            created_op = Config.INSTANCE.create()
    
          created_op.result(
    
                SPANNER_OPERATION_TIMEOUT_IN_SECONDS
            )  # block until completion
    

    tests/system/test_system_dbapi.py:99:


    .nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:130: in result
    self._blocking_poll(timeout=timeout, **kwargs)


    self = <google.api_core.operation.Operation object at 0x7f151c7e5e20>
    timeout = 60, retry = <google.api_core.retry.Retry object at 0x7f151e5f6190>

    def _blocking_poll(self, timeout=None, retry=DEFAULT_RETRY):
        """Poll and wait for the Future to be resolved.
    
        Args:
            timeout (int):
                How long (in seconds) to wait for the operation to complete.
                If None, wait indefinitely.
        """
        if self._result_set:
            return
    
        retry_ = self._retry.with_deadline(timeout)
    
        try:
            kwargs = {} if retry is DEFAULT_RETRY else {"retry": retry}
            retry_(self._done_or_raise)(**kwargs)
        except exceptions.RetryError:
    
          raise concurrent.futures.TimeoutError(
    
                "Operation did not complete within the designated " "timeout."
            )
    

    E concurrent.futures._base.TimeoutError: Operation did not complete within the designated timeout.

    .nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:110: TimeoutError

  6. removed
    priority: p1Important issue which blocks shipping the next release. Will be fixed prior to next release.
    on Jul 30, 2021
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Metadata

Metadata

Assignees

Labels

api: spannerIssues related to the googleapis/python-spanner API.flakybot: flakyTells the Flaky Bot not to close or comment on this issue.flakybot: issueAn issue filed by the Flaky Bot. Should not be added manually.type: processA process-related concern. May include testing, release, or the like.

Type

No type

Projects

No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions