-
Notifications
You must be signed in to change notification settings - Fork 7
feat(stovepipe): add the ValidationFact entity and its create-only queue-scoped store #552
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Merged
Changes from all commits
Commits
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,58 @@ | ||
| // Copyright (c) 2025 Uber Technologies, Inc. | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
|
|
||
| package entity | ||
|
|
||
| // Degree bounds. A degree answers "how broken was this scope?" on a closed | ||
| // interval, so the two endpoints are named and the values between them are | ||
| // meaningful rather than arbitrary. | ||
| const ( | ||
| // DegreeGreen is a fully healthy scope: nothing broken. | ||
| DegreeGreen = 0.0 | ||
| // DegreeBroken is a fully broken scope. | ||
| DegreeBroken = 1.0 | ||
| ) | ||
|
|
||
| // ValidationFact is the durable record of how broken one scope was at one commit. | ||
| // | ||
| // It is immutable and create-only: the first fact written for an identity is the | ||
| // permanent answer, and a later contradicting outcome for the same identity is | ||
| // dropped rather than overwriting it. Identity is (queue, URI, project) — the | ||
| // queue is the binding of the store the fact lives in and so is not carried on | ||
| // the fact itself. | ||
| // | ||
| // Absence is distinct from DegreeGreen: a URI with no fact has not been | ||
| // validated, which is not the same as having been validated and found healthy. | ||
| type ValidationFact struct { | ||
| // URI is the commit the fact describes, in the queue's VCS-agnostic locator | ||
| // form. Unique per (queue, URI, project). | ||
| URI string `json:"uri"` | ||
| // Project is the scope the degree applies to. Empty means the whole | ||
| // repository; named projects narrow the fact to part of it. | ||
| Project string `json:"project"` | ||
| // Degree is how broken the scope was, on the closed interval | ||
| // [DegreeGreen, DegreeBroken]. Intermediate values describe partial | ||
| // breakage. | ||
| Degree float64 `json:"degree"` | ||
| // RequestID is the request that established the fact. | ||
| RequestID string `json:"request_id"` | ||
| // CreatedAt is the millisecond timestamp at which the fact was recorded. | ||
| CreatedAt int64 `json:"created_at"` | ||
| } | ||
|
|
||
| // IsGreen reports whether the fact records a fully healthy scope. Any degree | ||
| // above DegreeGreen is some amount of broken. | ||
| func (f ValidationFact) IsGreen() bool { | ||
| return f.Degree == DegreeGreen | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,41 @@ | ||
| // Copyright (c) 2025 Uber Technologies, Inc. | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
|
|
||
| package entity | ||
|
|
||
| import ( | ||
| "testing" | ||
|
|
||
| "github.com/stretchr/testify/assert" | ||
| ) | ||
|
|
||
| func TestValidationFact_IsGreen(t *testing.T) { | ||
| tests := []struct { | ||
| name string | ||
| degree float64 | ||
| expected bool | ||
| }{ | ||
| {name: "green degree is green", degree: DegreeGreen, expected: true}, | ||
| {name: "broken degree is not green", degree: DegreeBroken, expected: false}, | ||
| {name: "partial breakage is not green", degree: 0.25, expected: false}, | ||
| {name: "zero value is green", degree: 0, expected: true}, | ||
| } | ||
|
|
||
| for _, tt := range tests { | ||
| t.Run(tt.name, func(t *testing.T) { | ||
| fact := ValidationFact{Degree: tt.degree} | ||
| assert.Equal(t, tt.expected, fact.IsGreen()) | ||
| }) | ||
| } | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
71 changes: 71 additions & 0 deletions
71
stovepipe/extension/storage/mock/validation_fact_store_mock.go
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
15 changes: 15 additions & 0 deletions
15
stovepipe/extension/storage/mysql/schema/validation_fact.sql
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,15 @@ | ||
| -- validation_fact holds the immutable record of how broken one scope was at one commit. | ||
| -- The composite PK (queue, uri, project) is the fact's identity and makes the write | ||
| -- create-only: a second insert for the same identity raises a duplicate-key error rather | ||
| -- than overwriting the first fact. queue leads the PK so the table is shardable by queue. | ||
| -- project is empty for a whole-repository fact and names a narrower scope otherwise; it is | ||
| -- part of the key rather than a nullable attribute so both kinds of fact coexist per commit. | ||
| CREATE TABLE IF NOT EXISTS validation_fact ( | ||
| queue VARCHAR(255) NOT NULL, | ||
| uri VARCHAR(255) NOT NULL, | ||
| project VARCHAR(255) NOT NULL DEFAULT '', | ||
| degree DOUBLE NOT NULL, | ||
| request_id VARCHAR(255) NOT NULL, | ||
| created_at BIGINT NOT NULL, | ||
| PRIMARY KEY (queue, uri, project) | ||
| ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
95 changes: 95 additions & 0 deletions
95
stovepipe/extension/storage/mysql/validation_fact_store.go
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,95 @@ | ||
| // Copyright (c) 2025 Uber Technologies, Inc. | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
|
|
||
| package mysql | ||
|
|
||
| import ( | ||
| "context" | ||
| "database/sql" | ||
| "errors" | ||
| "fmt" | ||
|
|
||
| "github.com/uber-go/tally" | ||
| "github.com/uber/submitqueue/platform/metrics" | ||
| "github.com/uber/submitqueue/stovepipe/entity" | ||
| "github.com/uber/submitqueue/stovepipe/extension/storage" | ||
| ) | ||
|
|
||
| type validationFactStore struct { | ||
| db *sql.DB | ||
| scope tally.Scope | ||
| // queue is the queue name this store instance is bound to; every read and | ||
| // write is scoped to it. | ||
| queue string | ||
| } | ||
|
|
||
| // NewValidationFactStore creates a new MySQL-backed ValidationFactStore. | ||
| func NewValidationFactStore(db *sql.DB, scope tally.Scope, queue string) storage.ValidationFactStore { | ||
| return &validationFactStore{db: db, scope: scope, queue: queue} | ||
| } | ||
|
|
||
| // Create writes one immutable fact for the bound queue. Returns ErrAlreadyExists if the | ||
| // queue already holds a fact for the (uri, project) pair. | ||
| func (v *validationFactStore) Create(ctx context.Context, fact entity.ValidationFact) (retErr error) { | ||
| op := metrics.Begin(v.scope, "create", metrics.StorageLatencyBuckets) | ||
| defer func() { op.Complete(retErr) }() | ||
|
|
||
| queue := v.queue | ||
| _, err := v.db.ExecContext(ctx, | ||
| `INSERT INTO validation_fact (queue, uri, project, degree, request_id, created_at) | ||
| VALUES (?, ?, ?, ?, ?, ?)`, | ||
| queue, | ||
| fact.URI, | ||
| fact.Project, | ||
| fact.Degree, | ||
| fact.RequestID, | ||
| fact.CreatedAt, | ||
| ) | ||
| if err != nil { | ||
| if isDuplicateEntry(err) { | ||
| return fmt.Errorf("validation_fact queue=%s uri=%s project=%s: %w", queue, fact.URI, fact.Project, storage.ErrAlreadyExists) | ||
| } | ||
| return fmt.Errorf("failed to insert validation fact queue=%s uri=%s project=%s: %w", queue, fact.URI, fact.Project, err) | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| // Get returns the bound queue's fact for uri and project. Returns ErrNotFound if absent. | ||
| func (v *validationFactStore) Get(ctx context.Context, uri, project string) (ret entity.ValidationFact, retErr error) { | ||
| op := metrics.Begin(v.scope, "get", metrics.StorageLatencyBuckets) | ||
| defer func() { op.Complete(retErr) }() | ||
|
|
||
| queue := v.queue | ||
| var fact entity.ValidationFact | ||
| err := v.db.QueryRowContext(ctx, | ||
| "SELECT uri, project, degree, request_id, created_at FROM validation_fact WHERE queue = ? AND uri = ? AND project = ?", | ||
| queue, uri, project, | ||
| ).Scan( | ||
| &fact.URI, | ||
| &fact.Project, | ||
| &fact.Degree, | ||
| &fact.RequestID, | ||
| &fact.CreatedAt, | ||
| ) | ||
|
|
||
| if errors.Is(err, sql.ErrNoRows) { | ||
| return entity.ValidationFact{}, storage.WrapNotFound(err) | ||
| } | ||
| if err != nil { | ||
| return entity.ValidationFact{}, fmt.Errorf("failed to get validation fact queue=%s uri=%s project=%s from the database: %w", queue, uri, project, err) | ||
| } | ||
|
|
||
| return fact, nil | ||
| } |
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.