build: reduce build time (#1370) · googleapis/python-spanner@e53eaa2 · GitHub | Latest TMZ Celebrity News & Gossip | Watch TMZ Live
Skip to content

Commit e53eaa2

Browse files
olavloiteolavloite
authored
build: reduce build time (#1370)
* build: reduce build time * build: move some checks to GitHub Actions * test: speed up more tests
1 parent aae8d61 commit e53eaa2

24 files changed

+316
-121
lines changed

‎.github/workflows/mock_server_tests.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ on:
55
pull_request:
66
name: Run Spanner tests against an in-mem mock server
77
jobs:
8-
system-tests:
8+
mock-server-tests:
99
runs-on: ubuntu-latest
1010

1111
steps:

‎.github/workflows/presubmit.yaml

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
on:
2+
push:
3+
branches:
4+
- main
5+
pull_request:
6+
name: Presubmit checks
7+
permissions:
8+
contents: read
9+
pull-requests: write
10+
jobs:
11+
lint:
12+
runs-on: ubuntu-latest
13+
14+
steps:
15+
- name: Checkout code
16+
uses: actions/checkout@v4
17+
- name: Setup Python
18+
uses: actions/setup-python@v5
19+
with:
20+
python-version: 3.8
21+
- name: Install nox
22+
run: python -m pip install nox
23+
- name: Check formatting
24+
run: nox -s lint
25+
units:
26+
runs-on: ubuntu-latest
27+
strategy:
28+
fail-fast: false
29+
matrix:
30+
python: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]
31+
32+
steps:
33+
- name: Checkout code
34+
uses: actions/checkout@v4
35+
- name: Setup Python
36+
uses: actions/setup-python@v5
37+
with:
38+
python-version: ${{matrix.python}}
39+
- name: Install nox
40+
run: python -m pip install nox
41+
- name: Run unit tests
42+
run: nox -s unit-${{matrix.python}}

‎.kokoro/presubmit/presubmit.cfg

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
# Format: //devtools/kokoro/config/proto/build.proto
22

3-
# Disable system tests.
3+
# Only run a subset of all nox sessions
44
env_vars: {
5-
key: "RUN_SYSTEM_TESTS"
6-
value: "false"
5+
key: "NOX_SESSION"
6+
value: "unit-3.8 unit-3.12 cover docs docfx"
77
}

‎google/cloud/spanner_dbapi/connection.py

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@
1717

1818
from google.api_core.exceptions import Aborted
1919
from google.api_core.gapic_v1.client_info import ClientInfo
20+
from google.auth.credentials import AnonymousCredentials
21+
2022
from google.cloud import spanner_v1 as spanner
2123
from google.cloud.spanner_dbapi import partition_helper
2224
from google.cloud.spanner_dbapi.batch_dml_executor import BatchMode, BatchDmlExecutor
@@ -784,11 +786,15 @@ def connect(
784786
route_to_leader_enabled=route_to_leader_enabled,
785787
)
786788
else:
789+
client_options = None
790+
if isinstance(credentials, AnonymousCredentials):
791+
client_options = kwargs.get("client_options")
787792
client = spanner.Client(
788793
project=project,
789794
credentials=credentials,
790795
client_info=client_info,
791796
route_to_leader_enabled=route_to_leader_enabled,
797+
client_options=client_options,
792798
)
793799
else:
794800
if project is not None and client.project != project:

‎google/cloud/spanner_dbapi/transaction_helper.py

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -162,7 +162,7 @@ def add_execute_statement_for_retry(
162162
self._last_statement_details_per_cursor[cursor] = last_statement_result_details
163163
self._statement_result_details_list.append(last_statement_result_details)
164164

165-
def retry_transaction(self):
165+
def retry_transaction(self, default_retry_delay=None):
166166
"""Retry the aborted transaction.
167167
168168
All the statements executed in the original transaction
@@ -202,7 +202,9 @@ def retry_transaction(self):
202202
raise RetryAborted(RETRY_ABORTED_ERROR, ex)
203203
return
204204
except Aborted as ex:
205-
delay = _get_retry_delay(ex.errors[0], attempt)
205+
delay = _get_retry_delay(
206+
ex.errors[0], attempt, default_retry_delay=default_retry_delay
207+
)
206208
if delay:
207209
time.sleep(delay)
208210

‎google/cloud/spanner_v1/_helpers.py

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -510,6 +510,7 @@ def _metadata_with_prefix(prefix, **kw):
510510
def _retry_on_aborted_exception(
511511
func,
512512
deadline,
513+
default_retry_delay=None,
513514
):
514515
"""
515516
Handles retry logic for Aborted exceptions, considering the deadline.
@@ -520,7 +521,12 @@ def _retry_on_aborted_exception(
520521
attempts += 1
521522
return func()
522523
except Aborted as exc:
523-
_delay_until_retry(exc, deadline=deadline, attempts=attempts)
524+
_delay_until_retry(
525+
exc,
526+
deadline=deadline,
527+
attempts=attempts,
528+
default_retry_delay=default_retry_delay,
529+
)
524530
continue
525531

526532

@@ -608,7 +614,7 @@ def _metadata_with_span_context(metadata: List[Tuple[str, str]], **kw) -> None:
608614
inject(setter=OpenTelemetryContextSetter(), carrier=metadata)
609615

610616

611-
def _delay_until_retry(exc, deadline, attempts):
617+
def _delay_until_retry(exc, deadline, attempts, default_retry_delay=None):
612618
"""Helper for :meth:`Session.run_in_transaction`.
613619
614620
Detect retryable abort, and impose server-supplied delay.
@@ -628,15 +634,15 @@ def _delay_until_retry(exc, deadline, attempts):
628634
if now >= deadline:
629635
raise
630636

631-
delay = _get_retry_delay(cause, attempts)
637+
delay = _get_retry_delay(cause, attempts, default_retry_delay=default_retry_delay)
632638
if delay is not None:
633639
if now + delay > deadline:
634640
raise
635641

636642
time.sleep(delay)
637643

638644

639-
def _get_retry_delay(cause, attempts):
645+
def _get_retry_delay(cause, attempts, default_retry_delay=None):
640646
"""Helper for :func:`_delay_until_retry`.
641647
642648
:type exc: :class:`grpc.Call`
@@ -658,6 +664,8 @@ def _get_retry_delay(cause, attempts):
658664
retry_info.ParseFromString(retry_info_pb)
659665
nanos = retry_info.retry_delay.nanos
660666
return retry_info.retry_delay.seconds + nanos / 1.0e9
667+
if default_retry_delay is not None:
668+
return default_retry_delay
661669

662670
return 2**attempts + random.random()
663671

‎google/cloud/spanner_v1/batch.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -257,9 +257,11 @@ def commit(
257257
deadline = time.time() + kwargs.get(
258258
"timeout_secs", DEFAULT_RETRY_TIMEOUT_SECS
259259
)
260+
default_retry_delay = kwargs.get("default_retry_delay", None)
260261
response = _retry_on_aborted_exception(
261262
method,
262263
deadline=deadline,
264+
default_retry_delay=default_retry_delay,
263265
)
264266
self.committed = response.commit_timestamp
265267
self.commit_stats = response.commit_stats

‎google/cloud/spanner_v1/client.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -241,7 +241,9 @@ def __init__(
241241
meter_provider = MeterProvider(
242242
metric_readers=[
243243
PeriodicExportingMetricReader(
244-
CloudMonitoringMetricsExporter(),
244+
CloudMonitoringMetricsExporter(
245+
project_id=project, credentials=credentials
246+
),
245247
export_interval_millis=METRIC_EXPORT_INTERVAL_MS,
246248
)
247249
]

‎google/cloud/spanner_v1/metrics/metrics_exporter.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
from typing import Optional, List, Union, NoReturn, Tuple, Dict
2727

2828
import google.auth
29+
from google.auth import credentials as ga_credentials
2930
from google.api.distribution_pb2 import ( # pylint: disable=no-name-in-module
3031
Distribution,
3132
)
@@ -111,6 +112,7 @@ def __init__(
111112
self,
112113
project_id: Optional[str] = None,
113114
client: Optional["MetricServiceClient"] = None,
115+
credentials: Optional[ga_credentials.Credentials] = None,
114116
):
115117
"""Initialize a custom exporter to send metrics for the Spanner Service Metrics."""
116118
# Default preferred_temporality is all CUMULATIVE so need to customize
@@ -121,6 +123,7 @@ def __init__(
121123
transport=MetricServiceGrpcTransport(
122124
channel=MetricServiceGrpcTransport.create_channel(
123125
options=_OPTIONS,
126+
credentials=credentials,
124127
)
125128
)
126129
)

‎google/cloud/spanner_v1/session.py

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -461,6 +461,7 @@ def run_in_transaction(self, func, *args, **kw):
461461
reraises any non-ABORT exceptions raised by ``func``.
462462
"""
463463
deadline = time.time() + kw.pop("timeout_secs", DEFAULT_RETRY_TIMEOUT_SECS)
464+
default_retry_delay = kw.pop("default_retry_delay", None)
464465
commit_request_options = kw.pop("commit_request_options", None)
465466
max_commit_delay = kw.pop("max_commit_delay", None)
466467
transaction_tag = kw.pop("transaction_tag", None)
@@ -502,7 +503,11 @@ def run_in_transaction(self, func, *args, **kw):
502503
except Aborted as exc:
503504
del self._transaction
504505
if span:
505-
delay_seconds = _get_retry_delay(exc.errors[0], attempts)
506+
delay_seconds = _get_retry_delay(
507+
exc.errors[0],
508+
attempts,
509+
default_retry_delay=default_retry_delay,
510+
)
506511
attributes = dict(delay_seconds=delay_seconds, cause=str(exc))
507512
attributes.update(span_attributes)
508513
add_span_event(
@@ -511,7 +516,9 @@ def run_in_transaction(self, func, *args, **kw):
511516
attributes,
512517
)
513518

514-
_delay_until_retry(exc, deadline, attempts)
519+
_delay_until_retry(
520+
exc, deadline, attempts, default_retry_delay=default_retry_delay
521+
)
515522
continue
516523
except GoogleAPICallError:
517524
del self._transaction
@@ -539,7 +546,11 @@ def run_in_transaction(self, func, *args, **kw):
539546
except Aborted as exc:
540547
del self._transaction
541548
if span:
542-
delay_seconds = _get_retry_delay(exc.errors[0], attempts)
549+
delay_seconds = _get_retry_delay(
550+
exc.errors[0],
551+
attempts,
552+
default_retry_delay=default_retry_delay,
553+
)
543554
attributes = dict(delay_seconds=delay_seconds)
544555
attributes.update(span_attributes)
545556
add_span_event(
@@ -548,7 +559,9 @@ def run_in_transaction(self, func, *args, **kw):
548559
attributes,
549560
)
550561

551-
_delay_until_retry(exc, deadline, attempts)
562+
_delay_until_retry(
563+
exc, deadline, attempts, default_retry_delay=default_retry_delay
564+
)
552565
except GoogleAPICallError:
553566
del self._transaction
554567
add_span_event(

‎noxfile.py

Lines changed: 36 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -181,21 +181,6 @@ def install_unittest_dependencies(session, *constraints):
181181
# XXX: Dump installed versions to debug OT issue
182182
session.run("pip", "list")
183183

184-
# Run py.test against the unit tests with OpenTelemetry.
185-
session.run(
186-
"py.test",
187-
"--quiet",
188-
"--cov=google.cloud.spanner",
189-
"--cov=google.cloud",
190-
"--cov=tests.unit",
191-
"--cov-append",
192-
"--cov-config=.coveragerc",
193-
"--cov-report=",
194-
"--cov-fail-under=0",
195-
os.path.join("tests", "unit"),
196-
*session.posargs,
197-
)
198-
199184

200185
@nox.session(python=UNIT_TEST_PYTHON_VERSIONS)
201186
@nox.parametrize(
@@ -329,9 +314,12 @@ def system(session, protobuf_implementation, database_dialect):
329314
session.skip(
330315
"Credentials or emulator host must be set via environment variable"
331316
)
332-
# If POSTGRESQL tests and Emulator, skip the tests
333-
if os.environ.get("SPANNER_EMULATOR_HOST") and database_dialect == "POSTGRESQL":
334-
session.skip("Postgresql is not supported by Emulator yet.")
317+
if not (
318+
os.environ.get("SPANNER_EMULATOR_HOST") or protobuf_implementation == "python"
319+
):
320+
session.skip(
321+
"Only run system tests on real Spanner with one protobuf implementation to speed up the build"
322+
)
335323

336324
# Install pyopenssl for mTLS testing.
337325
if os.environ.get("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false") == "true":
@@ -365,7 +353,7 @@ def system(session, protobuf_implementation, database_dialect):
365353
"SKIP_BACKUP_TESTS": "true",
366354
},
367355
)
368-
if system_test_folder_exists:
356+
elif system_test_folder_exists:
369357
session.run(
370358
"py.test",
371359
"--quiet",
@@ -567,30 +555,32 @@ def prerelease_deps(session, protobuf_implementation, database_dialect):
567555
system_test_path = os.path.join("tests", "system.py")
568556
system_test_folder_path = os.path.join("tests", "system")
569557

570-
# Only run system tests if found.
571-
if os.path.exists(system_test_path):
572-
session.run(
573-
"py.test",
574-
"--verbose",
575-
f"--junitxml=system_{session.python}_sponge_log.xml",
576-
system_test_path,
577-
*session.posargs,
578-
env={
579-
"PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION": protobuf_implementation,
580-
"SPANNER_DATABASE_DIALECT": database_dialect,
581-
"SKIP_BACKUP_TESTS": "true",
582-
},
583-
)
584-
if os.path.exists(system_test_folder_path):
585-
session.run(
586-
"py.test",
587-
"--verbose",
588-
f"--junitxml=system_{session.python}_sponge_log.xml",
589-
system_test_folder_path,
590-
*session.posargs,
591-
env={
592-
"PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION": protobuf_implementation,
593-
"SPANNER_DATABASE_DIALECT": database_dialect,
594-
"SKIP_BACKUP_TESTS": "true",
595-
},
596-
)
558+
# Only run system tests for one protobuf implementation on real Spanner to speed up the build.
559+
if os.environ.get("SPANNER_EMULATOR_HOST") or protobuf_implementation == "python":
560+
# Only run system tests if found.
561+
if os.path.exists(system_test_path):
562+
session.run(
563+
"py.test",
564+
"--verbose",
565+
f"--junitxml=system_{session.python}_sponge_log.xml",
566+
system_test_path,
567+
*session.posargs,
568+
env={
569+
"PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION": protobuf_implementation,
570+
"SPANNER_DATABASE_DIALECT": database_dialect,
571+
"SKIP_BACKUP_TESTS": "true",
572+
},
573+
)
574+
elif os.path.exists(system_test_folder_path):
575+
session.run(
576+
"py.test",
577+
"--verbose",
578+
f"--junitxml=system_{session.python}_sponge_log.xml",
579+
system_test_folder_path,
580+
*session.posargs,
581+
env={
582+
"PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION": protobuf_implementation,
583+
"SPANNER_DATABASE_DIALECT": database_dialect,
584+
"SKIP_BACKUP_TESTS": "true",
585+
},
586+
)

0 commit comments

Comments
 (0)

TMZ Celebrity News – Breaking Stories, Videos & Gossip

Looking for the latest TMZ celebrity news? You've come to the right place. From shocking Hollywood scandals to exclusive videos, TMZ delivers it all in real time.

Whether it’s a red carpet slip-up, a viral paparazzi moment, or a legal drama involving your favorite stars, TMZ news is always first to break the story. Stay in the loop with daily updates, insider tips, and jaw-dropping photos.

🎥 Watch TMZ Live

TMZ Live brings you daily celebrity news and interviews straight from the TMZ newsroom. Don’t miss a beat—watch now and see what’s trending in Hollywood.