From 673a8aff6c73aa966459df6b634979a75d205034 Mon Sep 17 00:00:00 2001 From: iancooper Date: Mon, 7 Sep 2026 09:48:04 +0100 Subject: [PATCH] =?UTF-8?q?docs:=20013=20phase=202=20=E2=80=94=20PostgreSQ?= =?UTF-8?q?L=20for=20both=20transport=20and=20Outbox?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds contents/PostgreSQLTransportAndOutbox.md, the guide Docs#67 is owed: one PostgreSQL database as broker and Outbox, so the business write and the message announcing it commit together. Nine steps, a failures section naming the exceptions by their text, nested under PostgreSQL Message Broker. Every code block is compiled, and step 9's output is measured on a real run of the companion sample in BrighterCommand/Brighter#4304 — which extends the existing PostgresTaskQueue sample and reuses its receiver unchanged. Four defects repaired on the two pages this guide names as prerequisites, each found by compiling them and none in P0-2 or P0-4's scope: PostgresOutbox.md:61 CS1739 — GetDDL's parameter is binaryMessagePayload on PostgreSql, while MsSql, MySql and Sqlite spell it hasBinaryMessagePayload PostgresOutbox.md:68-95 the printed DDL is not what GetDDL emits at 10.7.0 PostgreSQLMessageBroker.md:91 CS1503 — the producer registry factory takes a PostgresMessagingGatewayConnection PostgreSQLMessageBroker.md:44 the queue DDL is not what the transport runs Design §4.1's first named failure was stale: the registration guard added in Brighter#3952 ships in 10.7.0, so a V10 reader meets a ConfigurationException from AddProducers rather than the sweeper's "No Async outbox defined." The page prints the measured exception and names the older one as its historical equivalent. Gates: link 160 -> 161, pagelint 158 -> 159 pages and 773 -> 772 warnings, shape 157 -> 158 with the widest section unmoved at 12 of 20, redirects, versioncheck and optioncheck all unmoved. --changed reported 17 code blocks strict, 0 errors. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_0146UueHL6H3zGBGTYwz7GtL --- SUMMARY.md | 1 + contents/PostgreSQLMessageBroker.md | 29 +- contents/PostgreSQLTransportAndOutbox.md | 358 +++++++++++++++++++ contents/PostgresOutbox.md | 60 ++-- spec/011-authoring_conventions/pagetypes.tsv | 1 + spec/013-howto_guides/tasks.md | 125 ++++++- 6 files changed, 525 insertions(+), 49 deletions(-) create mode 100644 contents/PostgreSQLTransportAndOutbox.md diff --git a/SUMMARY.md b/SUMMARY.md index 8cd3340..158bdee 100644 --- a/SUMMARY.md +++ b/SUMMARY.md @@ -69,6 +69,7 @@ * [Azure Service Bus Configuration](/contents/AzureServiceBusConfiguration.md) * [PostgreSQL Message Broker](/contents/PostgreSQLMessageBroker.md) * [PostgreSQL Broker Trade-Offs](/contents/PostgreSQLBrokerTradeOffs.md) + * [PostgreSQL for Transport and Outbox](/contents/PostgreSQLTransportAndOutbox.md) * [MSSQL Message Broker](/contents/MSSQLMessageBroker.md) * [GCP Pub/Sub Configuration](/contents/GcpPubSubConfiguration.md) * [RocketMQ Configuration](/contents/RocketMQConfiguration.md) diff --git a/contents/PostgreSQLMessageBroker.md b/contents/PostgreSQLMessageBroker.md index 3cc6148..dc95573 100644 --- a/contents/PostgreSQLMessageBroker.md +++ b/contents/PostgreSQLMessageBroker.md @@ -38,22 +38,23 @@ dotnet add package Paramore.Brighter.MessagingGateway.Postgres ### Database Table -Create the queue store table in your PostgreSQL database: +Brighter creates the queue store table for you when a publication or subscription sets `MakeChannels = OnMissingChannel.Create`. Set `OnMissingChannel.Validate` instead to manage the table yourself — this is the DDL Brighter runs, and the one to match: ```sql -CREATE TABLE IF NOT EXISTS {schema}.{queue_store_table} +CREATE TABLE IF NOT EXISTS "{schema}"."{queue_store_table}" ( - "id" BIGSERIAL PRIMARY KEY, - "queue" VARCHAR(255) NOT NULL, - "content" JSONB NOT NULL, - "visible_timeout" TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - "created_at" TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP + "id" BIGINT GENERATED ALWAYS AS IDENTITY, + "visible_timeout" TIMESTAMPTZ, + "queue" VARCHAR(255), + "content" JSON ); -CREATE INDEX IF NOT EXISTS idx_{queue_store_table}_queue_visible - ON {schema}.{queue_store_table}("queue", "visible_timeout"); +CREATE INDEX IF NOT EXISTS "{schema}_{queue_store_table}_queue_visible_timeout_idx" + ON "{schema}"."{queue_store_table}"("queue", "visible_timeout") INCLUDE ("id"); ``` +The `content` column is `JSONB` rather than `JSON` when the payload is binary — see `binaryMessagePayload` below. + **Index Requirements**: The index on `(queue, visible_timeout)` is critical for performance. --- @@ -63,9 +64,11 @@ CREATE INDEX IF NOT EXISTS idx_{queue_store_table}_queue_visible ### Basic Producer Setup ```csharp +using System.Collections.Generic; +using Microsoft.Extensions.DependencyInjection; using Paramore.Brighter; +using Paramore.Brighter.Extensions.DependencyInjection; using Paramore.Brighter.MessagingGateway.Postgres; -using Paramore.Brighter.PostgreSql; // Database configuration var postgresConfiguration = new RelationalDatabaseConfiguration( @@ -75,6 +78,10 @@ var postgresConfiguration = new RelationalDatabaseConfiguration( binaryMessagePayload: true // Use JSONB for better performance ); +// The gateway connection wraps that configuration; the producer registry takes this, +// not the configuration itself +var connection = new PostgresMessagingGatewayConnection(postgresConfiguration); + // Publication configuration var publications = new List { @@ -89,7 +96,7 @@ var publications = new List // Producer registry var producerRegistry = new PostgresProducerRegistryFactory( - postgresConfiguration, + connection, publications ).Create(); diff --git a/contents/PostgreSQLTransportAndOutbox.md b/contents/PostgreSQLTransportAndOutbox.md new file mode 100644 index 0000000..896486e --- /dev/null +++ b/contents/PostgreSQLTransportAndOutbox.md @@ -0,0 +1,358 @@ +--- +description: "One PostgreSQL database can be both your message broker and your Outbox, so the business write and the message announcing it commit in a single transaction." +layout: + description: + visible: false +--- + +# Use PostgreSQL for Both Transport and Outbox + +> **How-to** · Applies to **Brighter V10** · Prerequisites: [PostgreSQL Message Broker](/contents/PostgreSQLMessageBroker.md), [PostgreSQL Outbox](/contents/PostgresOutbox.md) + +One PostgreSQL database can be both your message broker and your Outbox, so the business write and the message announcing it commit in a single transaction. + +That is the whole reason to compose them. With a separate broker you write your row, commit, then send — and a crash in between loses the message. Put the queue store and the [Outbox](/contents/OutboxPattern.md) in the database you were already writing to and there is no "in between": one transaction covers your row and the message, and the Sweeper moves the message onto the queue afterwards. + +This guide assumes you have read the two pages in the banner above. It adds the part neither of them covers on its own — running both against one database. + +A working version of everything below is in the Brighter repository at +`Brighter/samples/TaskQueue/PostgresTaskQueue/GreetingsSenderWithOutbox/`, with the +consumer beside it in `GreetingsReceiverConsole/`. + +## Step 1: Install the Packages + +The transport, the Outbox, the transaction provider, the Sweeper and the provisioner are separate packages: + +```bash +dotnet add package Paramore.Brighter.MessagingGateway.Postgres +dotnet add package Paramore.Brighter.Outbox.PostgreSql +dotnet add package Paramore.Brighter.PostgreSql +dotnet add package Paramore.Brighter.Outbox.Hosting +dotnet add package Paramore.Brighter.BoxProvisioning.PostgreSql +``` + +`Paramore.Brighter.PostgreSql` is the one people miss. It holds `PostgreSqlConnectionProvider` and `PostgreSqlTransactionProvider`, which are what let your handler and the Outbox share a transaction — without it there is no composition, only two subsystems pointed at the same host. + +## Step 2: Create the Queue and Outbox Tables + +You need two Brighter tables plus your own. Brighter can create both of its own, by two different routes, and you can also drive the DDL yourself. + +**The queue store table** is created by the transport when a publication or subscription says `MakeChannels = OnMissingChannel.Create`. To manage it yourself, use `OnMissingChannel.Validate` and create it first — this is the DDL the transport itself runs, with `JSONB` in place of `JSON` when the payload is binary: + +```sql +CREATE TABLE IF NOT EXISTS "public"."Queue" +( + "id" BIGINT GENERATED ALWAYS AS IDENTITY, + "visible_timeout" TIMESTAMPTZ, + "queue" VARCHAR(255), + "content" JSON +); + +CREATE INDEX IF NOT EXISTS "public_Queue_queue_visible_timeout_idx" + ON "public"."Queue"("queue", "visible_timeout") INCLUDE ("id"); +``` + +**The Outbox table** is created and migrated by [Box Provisioning](/contents/BoxProvisioning.md) at startup, which is the call in step 8. To manage it yourself instead, ask Brighter for the same DDL and run it through your own tooling: + +```csharp +using Paramore.Brighter.Outbox.PostgreSql; + +// The DDL for a table that stores the message body as TEXT +string ddl = PostgreSqlOutboxBuilder.GetDDL("Outbox"); + +// Pass binaryMessagePayload: true for a BYTEA body +string binaryDdl = PostgreSqlOutboxBuilder.GetDDL("Outbox", binaryMessagePayload: true); +``` + +**The two tables do not agree about capitals, and this will bite you at a `psql` prompt.** The transport quotes the configured name as you wrote it, so `queueStoreTable: "Queue"` becomes a table called `Queue`. The Outbox lowercases the name and *then* quotes it, so `outBoxTableName: "Outbox"` becomes `outbox` — deliberately, so that a configured `"Outbox"` still matches the table older Brighter versions created unquoted. The consequence is that `select * from "Queue"` works and `select * from "Outbox"` returns `relation "Outbox" does not exist`. Query the Outbox unquoted, or as `"outbox"`. + +Your own tables are yours. Brighter does not create, migrate or know about them. + +## Step 3: Describe Both Tables in One Configuration + +This is the pivot the whole guide turns on. The queue store, the Outbox and the Inbox are three parameters on **one** object, so there is no second configuration to keep in step: + +```csharp +using Paramore.Brighter; +using Paramore.Brighter.MessagingGateway.Postgres; + +const string connectionString = + "Host=localhost;Port=5432;Username=postgres;Password=password;Database=brightertests"; + +// One object, both tables. Both names below are the defaults; naming them is the point. +var configuration = new RelationalDatabaseConfiguration( + connectionString, + outBoxTableName: "Outbox", + queueStoreTable: "Queue"); + +// The transport takes the same object, wrapped. PostgresMessagingGatewayConnection is a +// holder and adds no settings of its own. +var connection = new PostgresMessagingGatewayConnection(configuration); +``` + +For every option this type carries — including `schemaName`, `inboxTableName` and the payload flags — see [Relational Database Configuration Reference](/contents/RelationalDatabaseConfigurationReference.md#relational-database-configuration-options). + +**One flag means two things, because two subsystems read it.** `binaryMessagePayload` tells the transport to store the queue's `content` column as `JSONB` rather than `JSON`, and tells the Outbox to store its `Body` column as `bytea` rather than `text`. Sharing the object shares the flag. If you want JSONB on the queue without moving the Outbox to `bytea`, set it per publication and per subscription instead — `PostgresPublication.BinaryMessagePayload` and `PostgresSubscription.BinaryMessagePayload` are both nullable and both override the shared value. + +## Step 4: Register the Configuration + +Put the same object in the container: + +```csharp +using Microsoft.Extensions.DependencyInjection; +using Paramore.Brighter; + +builder.Services.AddSingleton(configuration); +``` + +**This line is easy to leave out and the failure lands nowhere near it.** `TransactionProvider` in the next step is given as a *type*, so the container activates `PostgreSqlTransactionProvider`, and its constructor asks for exactly this interface. Omit the registration and your application starts, provisions the Outbox, and only then throws — see [Failures](#postgresql-transport-and-outbox-failures). + +## Step 5: Wire the Producer and the Outbox + +The publication says where the event goes; the three lines after `ProducerRegistry` are what make the Outbox durable and transactional: + +```csharp +using Microsoft.Extensions.DependencyInjection; +using Paramore.Brighter; +using Paramore.Brighter.Extensions.DependencyInjection; +using Paramore.Brighter.MessagingGateway.Postgres; +using Paramore.Brighter.Outbox.PostgreSql; +using Paramore.Brighter.PostgreSql; + +var producerRegistry = new PostgresProducerRegistryFactory( + connection, + [ + new PostgresPublication + { + Topic = new RoutingKey("greeting.event"), + MakeChannels = OnMissingChannel.Create + } + ]).Create(); + +builder.Services + .AddBrighter() + .AddProducers(configure => + { + configure.ProducerRegistry = producerRegistry; + + configure.Outbox = new PostgreSqlOutbox(configuration); + configure.ConnectionProvider = typeof(PostgreSqlConnectionProvider); + configure.TransactionProvider = typeof(PostgreSqlTransactionProvider); + }) + .AutoFromAssemblies(); +``` + +`PostgresProducerRegistryFactory` takes the `PostgresMessagingGatewayConnection` from step 3, not the `RelationalDatabaseConfiguration` inside it. + +**The transaction provider is not optional here.** It is what fixes the transaction type Brighter matches the Outbox against, and `PostgreSqlOutbox` only satisfies that match when the provider is PostgreSQL's. Leave it out and registration itself fails, again in [Failures](#postgresql-transport-and-outbox-failures). + +## Step 6: Wire the Consumer + +The consumer reads from the same queue store table. Note the order: `AddConsumers` extends `IServiceCollection`, while `AddProducers` extends the builder it returns, so a consumer registration comes first and everything else chains off it. + +```csharp +using Microsoft.Extensions.DependencyInjection; +using Paramore.Brighter; +using Paramore.Brighter.MessagingGateway.Postgres; +using Paramore.Brighter.ServiceActivator.Extensions.DependencyInjection; +using Paramore.Brighter.ServiceActivator.Extensions.Hosting; + +var subscriptions = new Subscription[] +{ + new PostgresSubscription( + new SubscriptionName("paramore.example.greeting"), + new ChannelName("greeting.event"), + new RoutingKey("greeting.event"), + timeOut: TimeSpan.FromMilliseconds(2000), + messagePumpType: MessagePumpType.Reactor, + makeChannels: OnMissingChannel.Create) +}; + +builder.Services.AddConsumers(options => + { + options.Subscriptions = subscriptions; + options.DefaultChannelFactory = new PostgresChannelFactory(connection); + }) + .AutoFromAssemblies(); + +builder.Services.AddHostedService(); +``` + +**Use `PostgresSubscription`, not `Subscription`.** `PostgresChannelFactory` casts what it is given down to `PostgresSubscription` and throws `ConfigurationException` if the cast fails — a plain `Subscription` compiles and then dies when the Dispatcher starts reading. + +## Step 7: Deposit and Clear Inside Your Transaction + +Ask the transaction provider for the connection and the transaction rather than opening your own. That shared pair is what makes two writes one atomic act: + +```csharp +using System; +using System.Data.Common; +using System.Threading; +using System.Threading.Tasks; +using Paramore.Brighter; + +public class AddGreetingHandlerAsync : RequestHandlerAsync +{ + private readonly IAmATransactionConnectionProvider _transactionProvider; + private readonly IAmACommandProcessor _postBox; + + public AddGreetingHandlerAsync( + IAmATransactionConnectionProvider transactionProvider, + IAmACommandProcessor postBox) + { + _transactionProvider = transactionProvider; + _postBox = postBox; + } + + public override async Task HandleAsync( + AddGreeting addGreeting, + CancellationToken cancellationToken = default) + { + DbConnection connection = await _transactionProvider.GetConnectionAsync(cancellationToken); + DbTransaction transaction = await _transactionProvider.GetTransactionAsync(cancellationToken); + + try + { + // 1. Your write, to your table. + await using (DbCommand command = connection.CreateCommand()) + { + command.Transaction = transaction; + command.CommandText = "insert into Greeting (Message) values (@message)"; + + DbParameter message = command.CreateParameter(); + message.ParameterName = "message"; + message.Value = addGreeting.Greeting; + command.Parameters.Add(message); + + await command.ExecuteNonQueryAsync(cancellationToken); + } + + // 2. Brighter's write, to the Outbox, on that same transaction. Nothing has + // reached the queue store table yet. + await _postBox.DepositPostAsync( + new GreetingEvent(addGreeting.Greeting), + _transactionProvider, + cancellationToken: cancellationToken); + + // 3. Both, or neither. + await _transactionProvider.CommitAsync(cancellationToken); + } + catch (Exception) + { + await _transactionProvider.RollbackAsync(cancellationToken); + throw; + } + finally + { + _transactionProvider.Close(); + } + + return await base.HandleAsync(addGreeting, cancellationToken); + } +} +``` + +There is no `ClearOutboxAsync` call here on purpose: the message waits in the Outbox for the Sweeper. Call `ClearOutboxAsync` after the commit instead if you would rather dispatch immediately, at the cost of doing the send on the request thread. + +## Step 8: Run the Outbox Sweeper + +The Sweeper is a hosted service that wakes on a timer, finds undispatched messages and sends them — here, into the queue store table in the same database. Chain both calls off the registration in step 5: + +```csharp +using System; +using Paramore.Brighter.BoxProvisioning; +using Paramore.Brighter.BoxProvisioning.PostgreSql; +using Paramore.Brighter.Outbox.Hosting; + +builder.Services + .AddBrighter() + .AddProducers(configure => + { + // ... as in step 5 + }) + .AutoFromAssemblies() + + // Creates and migrates the Outbox table at startup. Needs rights to CREATE TABLE. + .UseBoxProvisioning(options => options.AddPostgreSqlOutbox(configuration)) + + .UseOutboxSweeper(options => + { + options.TimerInterval = 5; + options.MinimumMessageAge = TimeSpan.FromSeconds(5); + }); +``` + +Both Sweeper values above are the defaults. Together they mean a message is picked up on the first tick after it is five seconds old, so expect a five to ten second delay before it appears on the queue. + +Running more than one instance? Configure a [distributed lock](/contents/PostgresDistributedLock.md) so only one Sweeper runs at a time. + +## Step 9: Verify It Worked + +Send one message, then look at the three tables. The output below is from a real run of the companion sample. + +Immediately after the send, the Outbox holds the message and the queue does not: + +```sql +select messageid, topic, dispatched is not null as dispatched from outbox; +select id, queue from "Queue"; +``` + +Five to ten seconds later the Sweeper has dispatched it, and the sender logs both halves: + +```text +info: Paramore.Brighter.CommandProcessor[1620710603] + Found 1 to clear out of amount 100 +info: Paramore.Brighter.CommandProcessor[1310740404] + Decoupled invocation of message: Topic:greeting.event Id:01a07aff-20b1-722b-a12a-c5ecb3c466f9 +``` + +Now the same two queries show the message dispatched *and* sitting on the queue: + +```text + messageid | topic | dispatched +--------------------------------------+----------------+------------ + 01a07aff-20b1-722b-a12a-c5ecb3c466f9 | greeting.event | t + + id | queue +----+---------------- + 1 | greeting.event +``` + +Start the consumer and the row leaves the queue table: + +```text +Received Greeting. Message Follows +Hello from the sender +info: Paramore.Brighter.MessagingGateway.Postgres.PostgresMessageConsumer[1174086769] + PostgresPullMessageConsumer: Deleted the message 01a07aff-20b1-722b-a12a-c5ecb3c466f9 with receipt handle 1 on the queue greeting.event +``` + +**The check that matters is the failure case**, because it is the reason for all of this. Throw inside the handler between the two writes and the commit, and both disappear together — the row counts in your table and in `outbox` are unchanged, and nothing reaches the queue. + +## PostgreSQL Transport and Outbox Failures + +Two mistakes account for most of the traffic on this composition, and neither error message names the line you need to change. + +**`Unable to register outbox of type PostgreSqlOutbox - no transaction provider has been registered that matches the outbox's transaction type`** + +A `ConfigurationException`, thrown by `AddProducers` while your application is still starting. Brighter takes the transaction type from `TransactionProvider`, falling back to `InMemoryTransactionProvider` when you do not set one, and then checks that the Outbox you supplied actually implements the Outbox interfaces for *that* transaction type. `PostgreSqlOutbox` does not implement them for the in-memory transaction. Set both `ConnectionProvider` and `TransactionProvider` as step 5 shows. + +Older Brighter versions had no such check, and the mismatch surfaced much later as `InvalidOperationException: No Async outbox defined.` from the Sweeper. If you find that message in a search result, this registration guard is its modern equivalent — on V10 you will meet the `ConfigurationException` first. + +**`Unable to resolve service for type 'Paramore.Brighter.IAmARelationalDatabaseConfiguration' while attempting to activate 'Paramore.Brighter.PostgreSql.PostgreSqlTransactionProvider'`** + +You skipped step 4. What makes this one expensive is how healthy everything looks first: the host starts, the Outbox is provisioned — you will see `Provisioned Outbox 'Outbox' successfully` — and the exception arrives only on the first attempt to resolve a command processor, naming a type your code never mentions. Add the `AddSingleton` line from step 4. + +**`relation "Outbox" does not exist`** + +Not a wiring fault at all — the Outbox table is `outbox`, in lower case, for the reason in step 2. Your messages are there. + +## Further Reading + +- [PostgreSQL Message Broker](/contents/PostgreSQLMessageBroker.md) — the transport on its own, with every subscription and publication option +- [PostgreSQL Outbox](/contents/PostgresOutbox.md) — the Outbox on its own, including the Entity Framework Core provider +- [Relational Database Configuration Reference](/contents/RelationalDatabaseConfigurationReference.md) — every option on the configuration object step 3 builds +- [Outbox Pattern](/contents/OutboxPattern.md) — why an Outbox, and what it does and does not guarantee +- [Box Provisioning](/contents/BoxProvisioning.md) — startup provisioning and migration for the Outbox table +- [PostgreSQL Broker Trade-Offs](/contents/PostgreSQLBrokerTradeOffs.md) — when a table-based queue is the wrong answer +- [Postgres Distributed Lock](/contents/PostgresDistributedLock.md) — required once more than one instance runs a Sweeper diff --git a/contents/PostgresOutbox.md b/contents/PostgresOutbox.md index d2160b5..2a61f69 100644 --- a/contents/PostgresOutbox.md +++ b/contents/PostgresOutbox.md @@ -50,6 +50,8 @@ The PostgreSQL Outbox requires a specific table in your database to store messag The `PostgreSqlOutboxBuilder.GetDDL()` method creates the SQL script for you. You can execute this script against your database to create the outbox table. ```csharp +using Paramore.Brighter.Outbox.PostgreSql; + // The table name can be whatever you choose. string tableName = "Outbox"; @@ -58,42 +60,46 @@ string ddl = PostgreSqlOutboxBuilder.GetDDL(tableName); // The DDL for a table that stores the message body as BYTEA // Useful if your message body is binary -string binaryDdl = PostgreSqlOutboxBuilder.GetDDL(tableName, hasBinaryMessagePayload: true); +string binaryDdl = PostgreSqlOutboxBuilder.GetDDL(tableName, binaryMessagePayload: true); ``` +**The parameter is named `binaryMessagePayload` here.** The MSSQL, MySQL and SQLite builders spell the same argument `hasBinaryMessagePayload`, so copying a call from one of those pages gives you `CS1739`. + ### **Example SQL Script** -Running `PostgreSqlOutboxBuilder.GetDDL("Outbox")` will generate the following SQL script: +Running `PostgreSqlOutboxBuilder.GetDDL("Outbox")` against **Brighter V10 (10.7.0)** generates the following SQL script: ```sql -CREATE TABLE "Outbox" ( - "MessageId" VARCHAR(255) NOT NULL, - "Topic" VARCHAR(255) NOT NULL, - "MessageType" VARCHAR(32) NOT NULL, - "Timestamp" TIMESTAMPTZ(3) NOT NULL, - "CorrelationId" VARCHAR(255) NULL, - "ReplyTo" VARCHAR(255) NULL, - "ContentType" VARCHAR(128) NULL, - "PartitionKey" VARCHAR(255) NULL, - "WorkflowId" VARCHAR(255) NULL, - "JobId" VARCHAR(255) NULL, - "Dispatched" TIMESTAMPTZ(3) NULL, - "HeaderBag" TEXT NOT NULL, - "Body" TEXT NOT NULL , - "Source" VARCHAR(255) NULL, - "Type" VARCHAR(255) NULL, - "DataSchema" VARCHAR(255) NULL, - "Subject" VARCHAR(255) NULL, - "TraceParent" VARCHAR(255) NULL, - "TraceState" VARCHAR(255) NULL, - "Baggage" TEXT NULL, - "Created" TIMESTAMPTZ(3) NOT NULL DEFAULT NOW(), - "CreatedID" INT NOT NULL GENERATED ALWAYS AS IDENTITY, - UNIQUE("CreatedID"), - PRIMARY KEY ("MessageId") +CREATE TABLE IF NOT EXISTS "outbox" +( + Id bigserial PRIMARY KEY, + MessageId character varying(255) UNIQUE NOT NULL, + Topic character varying(255) NULL, + MessageType character varying(32) NULL, + Timestamp timestamptz NULL, + CorrelationId character varying(255) NULL, + ReplyTo character varying(255) NULL, + ContentType character varying(128) NULL, + PartitionKey character varying(128) NULL, + WorkflowId character varying(255) NULL, + JobId character varying(255) NULL, + Dispatched timestamptz NULL, + HeaderBag text NULL, + Body text NULL, + Source character varying (255) NULL, + Type character varying (255) NULL, + DataSchema character varying (255) NULL, + Subject character varying (255) NULL, + TraceParent character varying (255) NULL, + TraceState character varying (255) NULL, + Baggage text NULL, + DataRef character varying (255) NULL, + SpecVersion character varying (255) NULL ); ``` +**Note the table name.** You configured `"Outbox"` and the DDL emits `"outbox"`: Brighter lowercases the identifier and then quotes it, so that a configured mixed-case name still resolves to the table that older, unquoted DDL created. A `select` against `"Outbox"` therefore fails with `relation "Outbox" does not exist` while `select * from outbox` succeeds. + ## PostgreSQL Outbox Configuration To configure the PostgreSQL Outbox, you need to provide an outbox implementation in the `AddProducers` configuration when setting up Brighter. diff --git a/spec/011-authoring_conventions/pagetypes.tsv b/spec/011-authoring_conventions/pagetypes.tsv index 6a72a0f..7845a5f 100644 --- a/spec/011-authoring_conventions/pagetypes.tsv +++ b/spec/011-authoring_conventions/pagetypes.tsv @@ -156,3 +156,4 @@ contents/FirestoreOutbox.md Reference high "spec 012 D13, the Firestore Outbox; contents/SpannerOutbox.md Reference high "spec 012 D13, the Spanner Outbox; no options type of its own, links the relational reference" Reference Brighter V10 contents/FirestoreInbox.md Reference high "spec 012 D16, the Firestore Inbox; shares the Outbox configuration type" Reference Brighter V10 contents/SpannerInbox.md Reference high "spec 012 D16, the Spanner Inbox; no options type of its own, links the relational reference" Reference Brighter V10 +contents/PostgreSQLTransportAndOutbox.md How-to high "spec 013 P0-1, the PostgreSQL transport-and-Outbox composition guide; Step N headings, executed once" How-to Brighter V10 diff --git a/spec/013-howto_guides/tasks.md b/spec/013-howto_guides/tasks.md index 126a7eb..7631882 100644 --- a/spec/013-howto_guides/tasks.md +++ b/spec/013-howto_guides/tasks.md @@ -447,7 +447,7 @@ forbids showing deprecated patterns as current — a different reason, deliberat **Goal:** the guide Docs#67 is owed. **Eleven tasks. One PR.** ~330 lines, How-to, nested under `PostgreSQLMessageBroker.md`. -- [ ] **Task 2.1:** Ask the Q4 question, now that the PR exists +- [x] **Task 2.1:** Ask the Q4 question, now that the PR exists - Input: design §11 Q4 - Output: a ruling on whether P0-1 gets a compiled sample in `../Brighter/samples/` - Notes: **Deferred by agreement, not open.** A write to `../Brighter` is authorised **per @@ -456,7 +456,7 @@ forbids showing deprecated patterns as current — a different reason, deliberat and expect `build` to be a **coin-flip** (Brighter#4276) — re-run the job rather than pushing an empty commit. -- [ ] **Task 2.2:** Write the front matter, H1, banner and opening sentence +- [x] **Task 2.2:** Write the front matter, H1, banner and opening sentence - Input: design §4.1 - Output: quoted `description:` with `layout.description.visible: false`; H1 *Use PostgreSQL for Both Transport and Outbox*; the banner naming both prerequisites @@ -465,7 +465,7 @@ forbids showing deprecated patterns as current — a different reason, deliberat `pagelint.py --fix` will write the front matter *from* the sentence; it refuses if the sentence fails rule 7, which is the check. -- [ ] **Task 2.3:** Steps 1–2 — packages and the two tables' DDL +- [x] **Task 2.3:** Steps 1–2 — packages and the two tables' DDL - Input: `PostgresOutbox.md` §NuGet re-pinned; `PostgreSQLMessageBroker.md:39`; `PostgresOutbox.md:64` - Output: `## Step 1: Install the Packages`, `## Step 2: Create the Queue and Outbox Tables` @@ -473,7 +473,7 @@ forbids showing deprecated patterns as current — a different reason, deliberat a pin here is checked by nothing — grep the tools for this filename before assuming either way (a checker's inclusion list is where its unstated obligations live). -- [ ] **Task 2.4:** Step 3 — one `RelationalDatabaseConfiguration`, three tables +- [x] **Task 2.4:** Step 3 — one `RelationalDatabaseConfiguration`, three tables - Input: design §2.5; `src/Paramore.Brighter/RelationalDatabaseConfiguration.cs:21` - Output: `## Step 3: Describe Both Tables in One Configuration` - Notes: **This is the pivot the whole guide turns on** — `queueStoreTable`, @@ -482,7 +482,7 @@ forbids showing deprecated patterns as current — a different reason, deliberat the first two. **Link `RelationalDatabaseConfigurationReference.md` for the option table; never restate it.** -- [ ] **Task 2.5:** Step 4 — register `IAmARelationalDatabaseConfiguration`, and say why +- [x] **Task 2.5:** Step 4 — register `IAmARelationalDatabaseConfiguration`, and say why - Input: `PostgresOutbox.md:116`; Brighter #3721 / #3755 (closed, *not a bug*) and #4279 - Output: `## Step 4: Register the Configuration` - Notes: **the #3721 trap.** `TransactionProvider` is a **`Type`**, activated by the @@ -491,7 +491,7 @@ forbids showing deprecated patterns as current — a different reason, deliberat `GetRequiredService()` throws, naming a type the reader's code never mentions. Mirror **009 rung 3's sample**, per obligation 3. -- [ ] **Task 2.6:** Steps 5–6 — producer, Outbox and consumer +- [x] **Task 2.6:** Steps 5–6 — producer, Outbox and consumer - Input: design §7 examples 5 and 6; `PostgreSQLMessageBroker.md:145` - Output: `## Step 5: Wire the Producer and the Outbox`, `## Step 6: Wire the Consumer` - Notes: **`AddConsumers` extends `IServiceCollection`; `AddProducers` extends the @@ -499,7 +499,7 @@ forbids showing deprecated patterns as current — a different reason, deliberat else chains off it — `services.AddBrighter().AddProducers(…).AddConsumers(…)` is **`CS1929`**, and eleven blocks across eight pages get this wrong today. -- [ ] **Task 2.7:** Steps 7–8 — deposit/commit/clear, and the Sweeper +- [x] **Task 2.7:** Steps 7–8 — deposit/commit/clear, and the Sweeper - Input: `PostgreSQLMessageBroker.md:362` made runnable; `PostgresOutbox.md:169` - Output: `## Step 7: Deposit and Clear Inside Your Transaction`, `## Step 8: Run the Outbox Sweeper` @@ -508,7 +508,7 @@ forbids showing deprecated patterns as current — a different reason, deliberat omission was invisible until the sample was built. Link `PostgresDistributedLock.md` for multi-instance sweepers. -- [ ] **Task 2.8:** Step 9 — the verification step, measured on a real run +- [x] **Task 2.8:** Step 9 — the verification step, measured on a real run - Input: design §7 example 9 - Output: `## Step 9: Verify It Worked` — verification SQL and the **expected log lines** - Notes: **AC7, and it is the criterion with no tool behind it.** 009's AC7 was found unmet @@ -516,7 +516,7 @@ forbids showing deprecated patterns as current — a different reason, deliberat was **false three times of eleven**. **When a page makes a factual claim about the reader's machine, the claim needs a measurement, not a diagnosis.** Run it. -- [ ] **Task 2.9:** The failures section — the two exceptions, by their text +- [x] **Task 2.9:** The failures section — the two exceptions, by their text - Input: design §4.1's two named failures - Output: `## PostgreSQL Transport and Outbox Failures` - Notes: print the **exception text a reader will have searched for** — @@ -525,7 +525,7 @@ forbids showing deprecated patterns as current — a different reason, deliberat `ConnectionProvider` and `TransactionProvider` on `AddProducers`) and the missing `IAmARelationalDatabaseConfiguration` registration from task 2.5. -- [ ] **Task 2.10:** `SUMMARY.md`, `pagetypes.tsv`, *Further Reading*, and compile +- [x] **Task 2.10:** `SUMMARY.md`, `pagetypes.tsv`, *Further Reading*, and compile - Input: design §6's `## Transports` diff - Output: the nested entry under `PostgreSQLMessageBroker.md`; a `pagetypes.tsv` row appended; every block compiled @@ -533,7 +533,7 @@ forbids showing deprecated patterns as current — a different reason, deliberat *PostgreSQL for Transport and Outbox*, deliberately shorter than the H1. Entry and page in the **same commit**. -- [ ] **Task 2.11:** Gates, and assert the four that move +- [x] **Task 2.11:** Gates, and assert the four that move - Input: design §9's four-new-pages row, taken one page at a time - Output: link 160 → **161**, pagelint 158 → **159**, shape 157 → **158** with **widest unmoved at 12 of 20**, redirects **unmoved**, optioncheck **unmoved**; `--verify` after @@ -546,6 +546,109 @@ forbids showing deprecated patterns as current — a different reason, deliberat --- +## Phase 2 as executed — 2026-09-07 + +**All eleven tasks done in one PR.** Every gate landed where design §9 and task 2.11 predicted: +link **160 → 161**, pagelint pages **158 → 159**, shape **157 → 158** with the widest section +**unmoved at 12 of 20**, redirects unmoved at **77 entries / 7858 bytes**, versioncheck unmoved +at **18 pins across 5 pages**, optioncheck unmoved at **59 tables / 519 rows**. The warning +count fell **773 → 772**. The `--changed` scope line read **17 code block(s) strict** across +**3 documentation page(s)**, with 0 errors — the new page's 13 blocks plus the four repaired +below. + +**Q4 was answered YES**, which is why this phase has a Brighter PR beside it. + +### The companion sample — Brighter#4304 + +[Brighter#4304](https://github.com/BrighterCommand/Brighter/pull/4304) adds +`samples/TaskQueue/PostgresTaskQueue/GreetingsSenderWithOutbox/`, registered in +`Brighter.slnx` in the same PR. It **extends** rather than creates, which is `CLAUDE.md`'s +stated preference order: `Greetings` and `GreetingsReceiverConsole` are reused **unchanged**, +and the only edit to an existing file is the one line in `Brighter.slnx`. + +**It was run, not merely built.** Happy path: deposit, then `Found 1 to clear out of amount 100` +and `Decoupled invocation of message: Topic:greeting.event`, then the receiver prints the +greeting and logs `Deleted the message … on the queue greeting.event`. Unhappy path (`--fail`): +the row counts in `greeting` and `outbox` are both unchanged, so neither write survived. Those +runs are where every line of step 9 comes from. + +### Six findings the task list did not predict + +**A. Design §4.1's first named failure is stale, and the tell was that the probe printed a +different exception.** The page was to print `InvalidOperationException: No Async outbox +defined.` — Q&A #3795 verbatim, from `OutboxProducerMediator.cs:502`. Removing the two +providers and running produces something else entirely: +`ConfigurationException: Unable to register outbox of type PostgreSqlOutbox - no transaction +provider has been registered that matches the outbox's transaction type`, thrown from +`AddProducers` **at registration**. `AddProducers` takes the transaction type from +`TransactionProvider ?? InMemoryTransactionProvider` and refuses an Outbox that does not +implement `IAmAnOutboxSync<,>`/`IAmAnOutboxAsync<,>` for *that* type. The guard was added +**2026-01-19 in Brighter#3952** and ships in `10.7.0`, so a V10 reader cannot reach the +sweeper message by this route. **This is §13 Q3's shape exactly** — a finding taken from a +public thread, accurately quoted, and overtaken by a fix nobody re-checked. The page prints +the measured exception and names the older one as its historical equivalent, because that is +the string a search engine still carries. + +**B. Four defects on the two pages P0-1 names as prerequisites, none of them in P0-2 or +P0-4's scope.** Ruled into this PR on 2026-09-07 rather than recorded, on design §11 Q6's own +principle that a guide cannot honestly link a page whose code does not compile: + +| Page | Site | The corpus said | The compiler says | +|---|---|---|---| +| `PostgresOutbox.md` | 61 | `GetDDL(tableName, hasBinaryMessagePayload: true)` | **`CS1739`** — PostgreSql's parameter is `binaryMessagePayload` | +| `PostgresOutbox.md` | 68–95 | an "Example SQL Script" `GetDDL("Outbox")` generates | a different table — `"outbox"`, `Id bigserial PRIMARY KEY`, `DataRef`, `SpecVersion`, no `Created`/`CreatedID` | +| `PostgreSQLMessageBroker.md` | 91 | `PostgresProducerRegistryFactory(postgresConfiguration, …)` | **`CS1503`** — it takes a `PostgresMessagingGatewayConnection` | +| `PostgreSQLMessageBroker.md` | 44–55 | `BIGSERIAL PRIMARY KEY`, `TIMESTAMP … DEFAULT`, `JSONB NOT NULL` | `BIGINT GENERATED ALWAYS AS IDENTITY`, `TIMESTAMPTZ`, `JSON`, and an index with `INCLUDE ("id")` | + +**C. `hasBinaryMessagePayload` is real — on the other three stores — which is what made row 1 +invisible.** MsSql, MySql and Sqlite all spell it `hasBinaryMessagePayload`; PostgreSql alone +spells it `binaryMessagePayload`. A `git grep -c` for the wrong name returns **3 files**, not +zero, so the cheap check exonerates it. **Four of five following a convention is what hides the +fifth** — the same shape as design §11 Q6's `MsSqlEntityFrameworkCoreTransactionProvider`, and +it was settled by compiling the published line and getting `CS1739`, with the corrected line +compiling clean as the control. `PostgresOutbox.md` now says so in a sentence, so the next +person to copy a sibling's call meets a warning instead of the compiler. + +**D. The two subsystems disagree about identifier case, on one configuration object.** Measured +on a real run: the queue table is `Queue` and the Outbox table is `outbox`. The transport quotes +the configured name as written (`PostgresMessagingGateway.cs`), while the Outbox lowercases and +*then* quotes (`PgIdentifier.Quote`), deliberately, so a configured `"Outbox"` still matches the +table older unquoted DDL created. So `select * from "Queue"` works and `select * from "Outbox"` +returns `relation "Outbox" does not exist`. **Only composing the two surfaces this**, which is +the argument for the guide existing; it is in step 2, in the failures section, and now in +`PostgresOutbox.md` as well. + +**E. One flag means two things once the object is shared.** `binaryMessagePayload` tells the +transport to store the queue's `content` as `JSONB` rather than `JSON`, and tells the Outbox to +store `Body` as `bytea` rather than `text`. `PostgreSQLMessageBroker.md` recommends `true` for +performance, and a reader sharing that object moves their Outbox to `bytea` without being told. +The escape is per-publication and per-subscription: `PostgresPublication.BinaryMessagePayload` +and `PostgresSubscription.BinaryMessagePayload` are both nullable and both override the shared +value (`PostgresMessageProducerFactory.cs:30`, `PostgresChannelFactory.cs:25`). Step 3 says so. + +**F. `GetDDL`'s output differs between `10.7.0` and `origin/master`, and the repair was one +paste away from documenting an unreleased column.** Run against `origin/master` by +`ProjectReference` it emits `CausationId` and a `idx_outbox_causationid` index; run against the +**released 10.7.0 package** it emits neither. Row 2 of finding B was repaired from the second, +by adding a `PackageReference` to `Paramore.Brighter.Outbox.PostgreSql 10.7.0` and printing it. +**A compile harness wired to `src/` measures the product's future**, and every page here +documents its present. + +### Two decisions worth naming + +- **The page carries no version pins**, which departs from task 2.3's instruction to pin against + `10.7.0`. That note also said to grep the tools first, and the grep is why: `versioncheck.py` + scans only the five pages in `TUTORIAL_PAGES`, so a pin here would be checked by nothing and + would rot silently, while adding the page to `TUTORIAL_PAGES` would move a gate design §9 + predicts unmoved. Both prerequisite pages name packages without versions and the banner + already says **Brighter V10**. +- **Step 2 documents both provisioning routes rather than choosing.** The queue table arrives + from `OnMissingChannel.Create` and the Outbox from `UseBoxProvisioning`, which is what the + sample does and is genuinely two routes in one database; `PostgresOutbox.md`'s Option A / + Option B framing is linked rather than restated. + +--- + ## Phase 3 — P0-3, `contents/HandlingPoisonMessages.md` **Goal:** the route a reader takes to get a poison message off the channel. **Seven tasks. One