diff --git a/.env.example b/.env.example index 4b11f5a..ef03a88 100644 --- a/.env.example +++ b/.env.example @@ -61,10 +61,19 @@ TASKDECK_STATUS_TOKEN= # unset it. A page includes contacts, API key hashes, and client webhook secrets. RELAY_TRANSFER_TOKEN= -# Inbound mail. The drains that read these run under the `ingress` compose -# profile and stay stopped until the queue URLs are set. +# Inbound mail. Deployed environments start the drain unconditionally as +# `relay-inbound-ingress`; it is this queue URL that decides whether there is +# anything to drain, and an unset URL is a drain that waits rather than fails. +# The local docker-compose stack has no inbound service. SQS_INBOUND_EMAIL_QUEUE_URL= +# Optional. When set, a filed message is also announced on this topic. The +# event names the stored message rather than carrying the body, so a consumer +# reads it from Relay. When unset, inbound mail is only filed. INBOUND_EMAIL_EVENTS_TOPIC_ARN= -INBOUND_EMAIL_IDEMPOTENCY_TABLE= INBOUND_EMAIL_ARTIFACT_PREFIX=processed/ +# Fallback routing, used only when no row in `inbound_addresses` matches. +# `python manage.py provision_inbound_addresses` copies these into that table. INBOUND_EMAIL_ROUTES= +# INBOUND_EMAIL_IDEMPOTENCY_TABLE is gone. Duplicate delivery is now suppressed +# by a unique constraint on inbound_messages.message_id, so idempotency needs no +# DynamoDB table and cannot half-succeed. diff --git a/README.md b/README.md index 00fc44e..0bb9210 100644 --- a/README.md +++ b/README.md @@ -101,6 +101,37 @@ uv run python manage.py check uv run pytest ``` +## Receive mail + +Relay files accepted inbound mail in its own mailbox instead of publishing it +onwards and keeping nothing. A message becomes an `inbound_messages` row: +headers, a snippet and the SES verdicts in Postgres, bodies and attachments in +object storage. Bodies stay in S3 because inbound mail is unbounded in size and +retention, and the disk fill that once disqualified inbound in production was a +host-disk problem. + +Create a receiving address in the console under **Configure > Receiving +addresses**. That is the whole change: no Terraform edit, no apply, no redeploy. +An address is a row, not a forwarding rule in another repository. + +**Mark as spam** on a message blocks the sender, and the block is checked +*before* storage, so future mail from that sender is discarded rather than filed +and filtered later. A discarded message is recorded as `blocked` with no body, so +the block is auditable and unblocking does not resurrect something nobody read. +Blocking the sender or the whole sending domain are separate buttons: one +correspondent at a shared domain can be blocked alone. + +Until a domain's SES receipt rule points at Relay, nothing arrives. Addresses +fall back to `INBOUND_EMAIL_ROUTES` when no row in `inbound_addresses` matches, so +an estate that has created no addresses is not silently dark; +`python manage.py provision_inbound_addresses` copies the environment routes into +the table. + +The `inbound-email` SNS event still publishes when +`INBOUND_EMAIL_EVENTS_TOPIC_ARN` is set, but it is smaller than it was: it names +the stored message instead of carrying the body and every attachment. See +[docs/worker-contracts.md](docs/worker-contracts.md). + ## Deploy the sandbox Pushing `main` runs the test suite and deploys the complete release through diff --git a/docs/design-system.md b/docs/design-system.md index c90b209..9245c2f 100644 --- a/docs/design-system.md +++ b/docs/design-system.md @@ -8,9 +8,15 @@ We use our own lightweight CSS system. Do not add Bootstrap, Tailwind, React, a Use these references for UI elements and interaction quality: +- DataOps design system (`../dataops/frontend/DESIGN_SYSTEM.md`) and the + Course Management Platform system it builds on: the primary visual + reference. GitHub Primer neutrals, restrained borders, compact row-based + content, content-width actions, and the CMP blue accent. - GitHub Primer: primary reference for product UI foundations, compact navigation, subdued surfaces, tables, forms, labels, focus states, and status treatment. - Shopify Polaris: secondary reference for admin workflow discipline, especially promoted filters vs advanced filters and clear page actions. - Resend: secondary reference for API docs, API keys, developer-facing settings, sparse examples, and transactional email vocabulary. +- Dapier ("Operator's Ledger") is a sibling look, not the reference here; keep + Datamailer aligned with DataOps. Use Postmark only for product concepts, not visual styling: @@ -73,20 +79,28 @@ All reusable styling must flow through tokens before page-specific CSS is added. ### Color Tokens -Core surface and text tokens: +Core surface and text tokens (values follow the DataOps/Primer palette; the +light page canvas is white, the sidebar and secondary surfaces use `#f6f8fa`, +and the action/link accent is CMP blue `#315f8f`): - `--dm-color-text` +- `--dm-color-heading` - `--dm-color-muted` +- `--dm-color-faint` - `--dm-color-border` +- `--dm-color-border-strong` - `--dm-color-background` - `--dm-color-surface` -- `--dm-color-surface-strong` +- `--dm-color-surface-strong` (hover/active tone) +- `--dm-color-accent-soft` (selected navigation and focus wash) - `--dm-color-focus` Action tokens: -- `--dm-color-primary` +- `--dm-color-primary` (filled controls; dark mode uses a lighter fill) - `--dm-color-primary-hover` +- `--dm-color-link` (links and selected-nav text; stays readable in dark mode) +- `--dm-color-link-hover` - `--dm-color-on-primary` State tokens: @@ -101,8 +115,12 @@ State tokens: - `--dm-color-danger-hover` - `--dm-color-danger-surface` - `--dm-color-danger-border` +- `--dm-color-info` +- `--dm-color-info-surface` +- `--dm-color-info-border` - `--dm-color-neutral` - `--dm-color-neutral-surface` +- `--dm-color-neutral-border` Do not use raw hex values outside `:root` unless there is a documented exception. @@ -124,14 +142,19 @@ Do not introduce one-off spacing values for page layout. If a repeated spacing n - `--dm-radius-sm`: controls, badges, nav items - `--dm-radius-md`: panels, empty states, table wrappers +- `--dm-font-sans`: Inter (self-hosted, SIL OFL) +- `--dm-font-mono`: IBM Plex Mono (self-hosted, SIL OFL); quantities, timings, and IDs render as data, not prose - `--dm-font-size-sm`: labels, help text, table headers - `--dm-font-size-base`: body and form controls - `--dm-font-size-lg`: section headings - `--dm-font-size-xl`: page headings - `--dm-control-height`: inputs and buttons - `--dm-content-width`: readable main-column width +- `--dm-sidebar-width`: persistent sidebar width Letter spacing stays normal. Font sizes do not scale with viewport width. +The shared component radius is 6px. Normal surfaces carry no shadow; shadows +are reserved for overlays. ## Component Contract @@ -176,12 +199,14 @@ Use these primitives before creating page-specific classes. ## Typography -- Use a system font stack. +- Use the self-hosted Inter stack (`--dm-font-sans`) for UI text and IBM Plex + Mono (`--dm-font-mono`) for quantities, timings, IDs, and code. Fonts must + not make third-party requests. - Keep letter spacing normal. - Do not scale font size with viewport width. -- Page titles should be clear but not hero-sized. -- Section headings should be compact. -- Table and metadata text should remain readable at 14-15px. +- Page titles are 32px semibold on desktop and 22px on mobile. +- Section headings are compact (16px semibold). +- Body and row text is 14px; table and metadata text may drop to 12-13px. - Help text should be short and muted. ## Color diff --git a/docs/worker-contracts.md b/docs/worker-contracts.md index 9f97013..33facb7 100644 --- a/docs/worker-contracts.md +++ b/docs/worker-contracts.md @@ -43,7 +43,9 @@ Transactional and campaign work use separate queues and workers so campaign back ## `inbound-email` v1 -SES stores the raw MIME object under the private inbound bucket's `raw/` prefix. S3 sends an object-created notification to the `inbound-email` queue. The Datamailer worker parses the message and publishes one normalized SNS event for each configured recipient route. +SES stores the raw MIME object under the private inbound bucket's `raw/` prefix. S3 sends an object-created notification to the `inbound-email` queue. Relay parses the message, files it in its own mailbox, and publishes one normalized SNS event naming the stored message. The published event is optional and off unless `INBOUND_EMAIL_EVENTS_TOPIC_ARN` is set. + +**The event is smaller than version 1 of this contract.** `body` and `attachments` are gone; the event carries `inbound_message_id` and a `raw_mime` reference instead, because the filed message is the record and a second full copy over SNS is the thing this path was built to stop doing. A consumer reading `body` or `attachments` must read them from Relay instead. See "The mailbox" below. The published event has this shape: @@ -51,41 +53,32 @@ The published event has this shape: { "contract": "inbound-email", "version": 1, - "event_id": "sha256-of-message-id-and-route", + "event_id": "", "event_type": "email.received", "occurred_at": "2026-07-12T07:30:01+00:00", - "route": "invoice", + "route": "invoice@mailer.dtcdev.click", "message_id": "", "sender": {"header": "Billing ", "addresses": ["billing@example.com"]}, - "recipients": { - "to": "invoice@mailer.dtcdev.click", - "cc": "", - "addresses": ["invoice@mailer.dtcdev.click"], - "matched": ["invoice@mailer.dtcdev.click"] - }, + "recipients": {"addresses": ["invoice@mailer.dtcdev.click"]}, "subject": "July invoice", - "date": "Sun, 12 Jul 2026 09:30:00 +0200", - "body": { - "text": {"content_type": "text/plain", "value": "Attached", "size": 8}, - "html": {"content_type": "text/html", "value": "

Attached

", "size": 15} - }, - "attachments": [{ - "filename": "invoice.pdf", - "content_type": "application/pdf", - "content_id": "", - "disposition": "attachment", - "size": 12345, - "s3": {"bucket": "private-inbound-bucket", "key": "processed/event-id/attachments/001-invoice.pdf"} - }], + "inbound_message_id": 41, "raw_mime": {"bucket": "private-inbound-bucket", "key": "raw/ses-object-key"} } ``` -Bodies up to `INBOUND_EMAIL_INLINE_BODY_MAX_BYTES` are included as text. Larger bodies, all attachments, and raw MIME are represented by private S3 references. Consumers need explicit read access to those object prefixes; Datamailer never publishes binary content through SNS. +The raw MIME and the extracted body parts are private S3 references, never inline values. Relay does not publish binary content through SNS. + +## The mailbox + +A message that matches a receiving address is stored as an `inbound_messages` row: headers, a snippet and the SES verdicts in Postgres, bodies and attachments in object storage. `inbound_message_id` is that row. Bodies stay in S3 because inbound mail is unbounded in size and retention. + +Receiving addresses come from the `inbound_addresses` table, created in the console, and fall back to `INBOUND_EMAIL_ROUTES` when the table has no match -- so the sandbox keeps working on the environment variable and a new address needs no deploy. `python manage.py provision_inbound_addresses` copies the environment routes into the table. + +`blocked_senders` is checked before storage. A message from a blocked address or domain is filed as `blocked` with no body stored, is not announced over SNS, and is not a read message. -Aliases are configured as exact address-to-route mappings in `INBOUND_EMAIL_ROUTES`, for example `invoice@mailer.dtcdev.click=invoice,todo@mailer.dtcdev.click=todo`. One message sent to aliases belonging to two routes produces two events. Duplicate delivery is suppressed by a DynamoDB conditional write on the SHA-256 of `Message-ID + route`. A failed SNS publish releases the claim so SQS can retry. +Duplicate delivery is suppressed by a unique constraint on `message_id`, which is the same guarantee the previous DynamoDB conditional write gave and no longer needs a service. A message with no `Message-ID` is rejected, because a sender that omits it has nothing stable to deduplicate on. One message sent to two managed addresses produces one row, filed under the first address that matched. -Raw MIME expires after `inbound_mail_retention_days` (60 days in the sandbox default). Extracted bodies and attachments expire after `inbound_email_artifact_retention_days` (14 days by default). Dapier must copy required artifacts to their system of record before expiry. +Raw MIME expires after `inbound_mail_retention_days` (60 days in the sandbox default, 14 in Relay production). Extracted body parts expire on the same lifecycle. A message that aged out of the queue is still in the bucket. Any external consumer must copy what it needs to its own system of record before expiry. ## `transactional-email` v1 diff --git a/mailing/admin.py b/mailing/admin.py index 19fd1eb..45913d3 100644 --- a/mailing/admin.py +++ b/mailing/admin.py @@ -3,6 +3,7 @@ from mailing.models import ( Audience, + BlockedSender, CallbackEndpoint, Campaign, CampaignRecipient, @@ -14,6 +15,8 @@ ContactTag, EmailEvent, EmailTemplate, + InboundAddress, + InboundMessage, MailchimpSync, MailchimpTagMapping, Organization, @@ -390,3 +393,29 @@ class MailchimpSyncAdmin(admin.ModelAdmin): "last_error", ) autocomplete_fields = ("contact", "client", "audience") + + +@admin.register(InboundAddress) +class InboundAddressAdmin(CreatedAtReadOnlyMixin, admin.ModelAdmin): + list_display = ("local_part", "domain", "is_active", "note", "created_at") + list_filter = ("is_active", "domain") + search_fields = ("local_part", "domain", "note") + + +@admin.register(InboundMessage) +class InboundMessageAdmin(CreatedAtReadOnlyMixin, admin.ModelAdmin): + list_display = ("sender_address", "subject", "recipient_address", "state", "created_at") + list_filter = ("state", "recipient_domain", "spam_verdict") + search_fields = ("subject", "snippet", "sender_address", "from_header", "recipient_address", "message_id") + readonly_fields = tuple( + field.name for field in InboundMessage._meta.fields if field.name not in {"id"} + ) + autocomplete_fields = ("inbound_address",) + + +@admin.register(BlockedSender) +class BlockedSenderAdmin(CreatedAtReadOnlyMixin, admin.ModelAdmin): + list_display = ("value", "scope", "origin", "created_at") + list_filter = ("scope", "origin") + search_fields = ("value", "reason") + autocomplete_fields = ("origin_message",) diff --git a/mailing/management/commands/provision_inbound_addresses.py b/mailing/management/commands/provision_inbound_addresses.py new file mode 100644 index 0000000..779d943 --- /dev/null +++ b/mailing/management/commands/provision_inbound_addresses.py @@ -0,0 +1,56 @@ +"""Create receiving addresses from the environment routes. + +The cutover path. A domain that was already receiving mail through +INBOUND_EMAIL_ROUTES has addresses the application does not know about, and +without this command the first message after the receipt rule is repointed +finds no managed address and is discarded as unmatched. + +Safe to run repeatedly. An address that already exists is left alone, so +retiring an address is not undone by running this again -- which matters, +because re-running it must not resurrect a deliberately retired address. + + python manage.py provision_inbound_addresses + python manage.py provision_inbound_addresses --dry-run +""" + +from django.conf import settings +from django.core.management.base import BaseCommand + +from mailing.services.inbound_views import backfill_from_routes + + +class Command(BaseCommand): + help = "Create a receiving address for every route in INBOUND_EMAIL_ROUTES." + + def add_arguments(self, parser): + parser.add_argument( + "--dry-run", + action="store_true", + help="Report what would be created without writing anything.", + ) + + def handle(self, *args, **options): + routes = settings.INBOUND_EMAIL_ROUTES + if not routes: + self.stdout.write( + self.style.WARNING( + "INBOUND_EMAIL_ROUTES is empty, so there is nothing to create. " + "Receiving addresses are managed in the console under " + "Configure > Receiving addresses." + ) + ) + return + + if options["dry_run"]: + self.stdout.write(f"{len(routes)} route(s) configured; would create the missing ones:") + for address in sorted(routes): + self.stdout.write(f" {address}") + return + + created = backfill_from_routes(routes) + if not created: + self.stdout.write(f"All {len(routes)} route(s) already have a receiving address.") + return + for row in created: + self.stdout.write(self.style.SUCCESS(f"Created {row.address}")) + self.stdout.write(f"{len(created)} address(es) created from INBOUND_EMAIL_ROUTES.") diff --git a/mailing/migrations/0028_blockedsender_inboundaddress_inboundmessage_and_more.py b/mailing/migrations/0028_blockedsender_inboundaddress_inboundmessage_and_more.py new file mode 100644 index 0000000..6c3beee --- /dev/null +++ b/mailing/migrations/0028_blockedsender_inboundaddress_inboundmessage_and_more.py @@ -0,0 +1,115 @@ +# Generated by Django 6.0.5 on 2026-09-29 01:00 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('mailing', '0027_callbackendpoint_clientcallback_and_more'), + ] + + operations = [ + migrations.CreateModel( + name='BlockedSender', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('scope', models.CharField(choices=[('address', 'Exact address'), ('domain', 'Whole domain')], default='address', max_length=10)), + ('value', models.CharField(max_length=320)), + ('reason', models.TextField(blank=True)), + ('origin', models.CharField(blank=True, max_length=32)), + ], + options={ + 'db_table': 'blocked_senders', + 'ordering': ['-created_at', '-id'], + }, + ), + migrations.CreateModel( + name='InboundAddress', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('local_part', models.CharField(max_length=64)), + ('domain', models.CharField(max_length=255)), + ('note', models.CharField(blank=True, max_length=255)), + ('is_active', models.BooleanField(default=True)), + ], + options={ + 'db_table': 'inbound_addresses', + 'ordering': ['domain', 'local_part'], + 'constraints': [models.UniqueConstraint(fields=('local_part', 'domain'), name='unique_inbound_address')], + }, + ), + migrations.CreateModel( + name='InboundMessage', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('message_id', models.CharField(blank=True, max_length=512)), + ('state', models.CharField(choices=[('received', 'Received'), ('blocked', 'Blocked sender'), ('read', 'Read')], default='received', max_length=20)), + ('subject', models.CharField(blank=True, max_length=998)), + ('snippet', models.TextField(blank=True)), + ('from_header', models.CharField(blank=True, max_length=998)), + ('sender_address', models.CharField(blank=True, db_index=True, max_length=320)), + ('sender_domain', models.CharField(blank=True, db_index=True, max_length=255)), + ('to_header', models.CharField(blank=True, max_length=998)), + ('cc_header', models.CharField(blank=True, max_length=998)), + ('recipient', models.CharField(blank=True, db_index=True, max_length=320)), + ('recipient_domain', models.CharField(blank=True, max_length=255)), + ('recipient_address', models.CharField(blank=True, max_length=320)), + ('sent_at', models.DateTimeField(blank=True, null=True)), + ('size_bytes', models.PositiveIntegerField(default=0)), + ('attachment_count', models.PositiveIntegerField(default=0)), + ('raw_bucket', models.CharField(blank=True, max_length=255)), + ('raw_key', models.CharField(blank=True, max_length=1024)), + ('body_text_key', models.CharField(blank=True, max_length=1024)), + ('body_html_key', models.CharField(blank=True, max_length=1024)), + ('spam_verdict', models.CharField(blank=True, max_length=64)), + ('virus_verdict', models.CharField(blank=True, max_length=64)), + ('spf_verdict', models.CharField(blank=True, max_length=64)), + ('dkim_verdict', models.CharField(blank=True, max_length=64)), + ('dmarc_verdict', models.CharField(blank=True, max_length=64)), + ('read_at', models.DateTimeField(blank=True, null=True)), + ('blocked_rule', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='blocked_messages', to='mailing.blockedsender')), + ('inbound_address', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='messages', to='mailing.inboundaddress')), + ], + options={ + 'db_table': 'inbound_messages', + 'ordering': ['-created_at', '-id'], + }, + ), + migrations.AddField( + model_name='blockedsender', + name='origin_message', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='origin_blocks', to='mailing.inboundmessage'), + ), + migrations.AddIndex( + model_name='inboundmessage', + index=models.Index(fields=['state', 'created_at'], name='inbound_msg_state_created_idx'), + ), + migrations.AddIndex( + model_name='inboundmessage', + index=models.Index(fields=['sender_address', 'created_at'], name='inbound_msg_sender_created_idx'), + ), + migrations.AddIndex( + model_name='inboundmessage', + index=models.Index(fields=['recipient', 'created_at'], name='inbound_msg_rcpt_created_idx'), + ), + migrations.AddConstraint( + model_name='inboundmessage', + constraint=models.UniqueConstraint(condition=models.Q(('message_id', ''), _negated=True), fields=('message_id',), name='unique_nonempty_inbound_message_id'), + ), + migrations.AddIndex( + model_name='blockedsender', + index=models.Index(fields=['scope', 'value'], name='blocked_sender_scope_value_idx'), + ), + migrations.AddConstraint( + model_name='blockedsender', + constraint=models.UniqueConstraint(fields=('scope', 'value'), name='unique_blocked_sender_scope_value'), + ), + ] diff --git a/mailing/models.py b/mailing/models.py index b0e7342..bc33912 100644 --- a/mailing/models.py +++ b/mailing/models.py @@ -1032,3 +1032,217 @@ class Meta: def __str__(self): return f"{self.email} {self.list_key} -> {self.tag} ({self.status})" + + +# --------------------------------------------------------------------------- +# Inbound mail +# +# Inbound mail is a mailbox, not a feed. The previous path parsed each message, +# published an event and kept nothing, which works for a downstream consumer with +# its own system of record but leaves nothing to read. A message is therefore +# stored here with its headers, a short text snippet, and a pointer to the raw +# MIME and the body parts in object storage. +# +# The bodies stay in S3 on purpose. Inbound mail is unbounded in size and +# retention, and the disk-fill that disqualified an inbound path in production +# was a host-disk problem. Headers are what the console lists, searches and +# sorts on, so they are cheap to index; bodies are read one at a time, on +# demand, by an operator who has already decided to open one. +# +# This is a receiving mailbox, not a sending one. Nothing here is a Contact and +# nothing here is deliverable, so a message is never suppressed, bounced or +# unsubscribed. +# --------------------------------------------------------------------------- + + +class InboundAddress(TimeStampedModel): + """An address Relay receives for, created in the application. + + This is the table that replaces the `local.email_forward_mapping` map in + main/aisl: adding an address is a row here rather than an edit to Terraform + and an apply. It is deliberately not a Contact and not a RecipientList -- + this address is where mail arrives, not who gets sent mail to. + + Not client-scoped. Inbound is a property of the Relay service, and a + receiving address does not belong to the tenant that happens to be selected + in the console. + """ + + local_part = models.CharField(max_length=64) + domain = models.CharField(max_length=255) + note = models.CharField(max_length=255, blank=True) + # Retired rather than deleted, so the address that a historical message was + # filed under still resolves to something. + is_active = models.BooleanField(default=True) + + class Meta: + db_table = "inbound_addresses" + ordering = ["domain", "local_part"] + constraints = [ + models.UniqueConstraint(fields=["local_part", "domain"], name="unique_inbound_address"), + ] + + def __str__(self): + return self.address + + @property + def address(self): + return f"{self.local_part}@{self.domain}" + + +class InboundMessageState(models.TextChoices): + RECEIVED = "received", "Received" + BLOCKED = "blocked", "Blocked sender" + READ = "read", "Read" + + +class InboundMessage(TimeStampedModel): + """One accepted inbound message, addressed to a Relay-owned address.""" + + # 512, not 998: a unique constraint on this column needs a btree index, and + # Postgres caps an index entry at 2704 bytes. 512 characters is 2048 bytes in + # the worst case, whereas 998 could reach 3992 and fail at insert time on a + # long id rather than at migration time. save() truncates. + message_id = models.CharField(max_length=512, blank=True) + state = models.CharField(max_length=20, choices=InboundMessageState.choices, default=InboundMessageState.RECEIVED) + + subject = models.CharField(max_length=998, blank=True) + snippet = models.TextField(blank=True) + + # The From header verbatim, and the address SES actually delivered to. The + # forwarding Lambda used to rewrite the header to "Sender via domain" and + # set Reply-To to the original sender; nothing rewrites anything here, so + # from_header stays what the sender claimed and reply_to stays empty. + from_header = models.CharField(max_length=998, blank=True) + sender_address = models.CharField(max_length=320, blank=True, db_index=True) + sender_domain = models.CharField(max_length=255, blank=True, db_index=True) + to_header = models.CharField(max_length=998, blank=True) + cc_header = models.CharField(max_length=998, blank=True) + + # The local part that matched, so an address created in the application is + # what a message is filed under, and so the list can be grouped by address + # without re-parsing a header. + recipient = models.CharField(max_length=320, blank=True, db_index=True) + recipient_domain = models.CharField(max_length=255, blank=True) + recipient_address = models.CharField(max_length=320, blank=True) + # The Relay-owned address this arrived for. SET_NULL rather than CASCADE: + # deleting an address should not delete the mail that was already received + # for it, which is the only record that it ever existed. + inbound_address = models.ForeignKey( + InboundAddress, + on_delete=models.SET_NULL, + null=True, + blank=True, + related_name="messages", + ) + + sent_at = models.DateTimeField(null=True, blank=True) + size_bytes = models.PositiveIntegerField(default=0) + attachment_count = models.PositiveIntegerField(default=0) + + # Raw MIME exactly as SES stored it, plus the extracted body parts. The + # console reads the body from the text pointer; nothing renders body_html + # without an operator deciding to. + raw_bucket = models.CharField(max_length=255, blank=True) + raw_key = models.CharField(max_length=1024, blank=True) + body_text_key = models.CharField(max_length=1024, blank=True) + body_html_key = models.CharField(max_length=1024, blank=True) + + # SES verdicts, kept because they are the cheapest available signal about + # whether to believe a message. x-ses-spam-verdict in particular is why a + # cold pitch can be separated from real mail without reading it. + spam_verdict = models.CharField(max_length=64, blank=True) + virus_verdict = models.CharField(max_length=64, blank=True) + spf_verdict = models.CharField(max_length=64, blank=True) + dkim_verdict = models.CharField(max_length=64, blank=True) + dmarc_verdict = models.CharField(max_length=64, blank=True) + + read_at = models.DateTimeField(null=True, blank=True) + blocked_rule = models.ForeignKey( + "BlockedSender", + on_delete=models.SET_NULL, + null=True, + blank=True, + related_name="blocked_messages", + ) + + class Meta: + db_table = "inbound_messages" + ordering = ["-created_at", "-id"] + indexes = [ + models.Index(fields=["state", "created_at"], name="inbound_msg_state_created_idx"), + models.Index(fields=["sender_address", "created_at"], name="inbound_msg_sender_created_idx"), + models.Index(fields=["recipient", "created_at"], name="inbound_msg_rcpt_created_idx"), + ] + constraints = [ + # One row per Message-ID, so a redelivery of the same message is + # recognised as a replay instead of filling the mailbox. Scoped to + # non-empty ids because many real senders omit the header and those + # messages are distinguished by bucket and key instead. + models.UniqueConstraint( + fields=["message_id"], + condition=~models.Q(message_id=""), + name="unique_nonempty_inbound_message_id", + ), + ] + + def __str__(self): + return f"{self.sender_address} -> {self.recipient_address}: {self.subject[:60]}" + + def save(self, *args, **kwargs): + # The unique constraint is on message_id, so a value longer than the + # column is truncated rather than a database error. A truncated id still + # deduplicates, and losing the tail of a malformed id costs nothing. + if len(self.message_id) > 512: + self.message_id = self.message_id[:512] + super().save(*args, **kwargs) + + @property + def is_spam(self): + return self.spam_verdict.strip().lower() == "yes" + + @property + def has_body(self): + return bool(self.body_text_key or self.body_html_key) + + +class BlockedSender(TimeStampedModel): + """A sender Relay refuses to file. + + One row per address or domain. The ingest path checks this before storing, + so blocking a sender discards future mail from it rather than filtering a + mailbox that already has to be read. + """ + + class Scope(models.TextChoices): + ADDRESS = "address", "Exact address" + DOMAIN = "domain", "Whole domain" + + scope = models.CharField(max_length=10, choices=Scope.choices, default=Scope.ADDRESS) + value = models.CharField(max_length=320) + reason = models.TextField(blank=True) + + # Set when an operator marks a message as spam, so the UI can show what a + # block was derived from and so the same sender is not blocked twice from + # two different messages. + origin_message = models.ForeignKey( + InboundMessage, + on_delete=models.SET_NULL, + null=True, + blank=True, + related_name="origin_blocks", + ) + origin = models.CharField(max_length=32, blank=True) + + class Meta: + db_table = "blocked_senders" + ordering = ["-created_at", "-id"] + constraints = [ + models.UniqueConstraint(fields=["scope", "value"], name="unique_blocked_sender_scope_value"), + ] + indexes = [ + models.Index(fields=["scope", "value"], name="blocked_sender_scope_value_idx"), + ] + + def __str__(self): + return f"{self.scope}: {self.value}" diff --git a/mailing/services/inbound_email.py b/mailing/services/inbound_email.py index 83d82f0..5a7e11a 100644 --- a/mailing/services/inbound_email.py +++ b/mailing/services/inbound_email.py @@ -1,172 +1,368 @@ +"""Filing accepted inbound mail into the mailbox. + +This module used to parse each message, extract its body and every attachment, +publish an `email.received` event and keep nothing. That was right for a +Datamailer-era consumer holding its own system of record, and wrong for a +mailbox, where the reader is a person in this console. The service name and the +`process_inbound_s3_notification` entry point are unchanged so the ingress drain +and the legacy Lambda handler keep working; what the function returns is +different. + +What changed, and why each part: + +- **Messages are stored.** Headers, a snippet and the SES verdicts go to + Postgres; bodies stay in object storage and are read on demand. Bodies stay in + S3 because inbound mail is unbounded in size and retention, and the disk fill + that disqualified inbound in production was a host-disk problem. + +- **The blocklist is consulted before storage, not after.** Blocking a sender + should stop future mail, not filter a mailbox that already has to be read, so a + blocked message is discarded at ingest and recorded as a blocked row rather + than a received one. That row is the audit trail for why a sender is blocked, + and it deliberately carries no body. + +- **Idempotency moved from DynamoDB to the message id.** The unique constraint + on a non-empty message_id is the same guarantee the DynamoDB conditional write + gave, inside the transaction that was already happening, with one fewer + service to fail. INBOUND_EMAIL_IDEMPOTENCY_TABLE is no longer read. + +- **Routes come from the InboundAddress table first.** Creating an address is a + row rather than an edit to Terraform and an apply, and the environment + variable is the fallback for the sandbox and for an estate that has created no + addresses yet. The database wins so an address retired in the console also + stops receiving, which a redeploy-free environment variable could not do. + +- **The publish contract got smaller, and that is a real change.** When + INBOUND_EMAIL_EVENTS_TOPIC_ARN is set the event is still published, but it now + identifies the stored message and its raw MIME instead of carrying the body + and every attachment. The stored message is the record, and a second full copy + over SNS was the thing this path was built to stop doing. A consumer of `body` + or `attachments` needs updating before the topic is enabled; see + docs/worker-contracts.md. +""" + import hashlib import json +import logging import re from datetime import UTC, datetime +from email.utils import parsedate_to_datetime from urllib.parse import unquote_plus from django.conf import settings -from django.core.exceptions import ImproperlyConfigured +from django.db import IntegrityError, transaction -from mailing.aws import aws_client, s3_client +from mailing.aws import s3_client from mailing.inbound_mime import parse_mime +from mailing.models import ( + BlockedSender, + InboundAddress, + InboundMessage, + InboundMessageState, +) + +logger = logging.getLogger(__name__) CONTRACT = "inbound-email" VERSION = 1 SAFE_FILENAME_RE = re.compile(r"[^A-Za-z0-9._-]+") +# Long enough to decide whether a pitch is a pitch, short enough that a list +# page does not carry megabytes of body text. +SNIPPET_CHARS = 500 +BLOCKED_SNIPPET_CHARS = 120 -def process_inbound_s3_notification(payload, *, s3=None, dynamodb=None, publisher=None): +def process_inbound_s3_notification(payload, *, s3=None, publisher=None): s3 = s3 or s3_client() - dynamodb = dynamodb or aws_client("dynamodb") - publisher = publisher or aws_client("sns") results = [] for record in payload.get("Records", []): if record.get("eventSource") != "aws:s3": raise ValueError("inbound worker accepts only S3 event records") bucket = record["s3"]["bucket"]["name"] key = unquote_plus(record["s3"]["object"]["key"]) - results.extend(process_inbound_object(bucket, key, s3=s3, dynamodb=dynamodb, publisher=publisher)) + results.append(process_inbound_object(bucket, key, s3=s3, publisher=publisher)) return results -def process_inbound_object(bucket, key, *, s3, dynamodb, publisher): +def process_inbound_object(bucket, key, *, s3, publisher=None): raw = s3.get_object(Bucket=bucket, Key=key)["Body"].read() message = parse_mime(raw) + if not message.message_id: raise ValueError("inbound MIME message is missing Message-ID") - routes = matching_routes(message.recipient_addresses, settings.INBOUND_EMAIL_ROUTES) - results = [] - for route, matched_recipients in routes.items(): - event_id = hashlib.sha256(f"{message.message_id}\0{route}".encode()).hexdigest() - if not claim_event(dynamodb, event_id, route, message.message_id): - results.append({"event_id": event_id, "route": route, "idempotent_replay": True}) + recipient = match_recipient(message.recipient_addresses) + if recipient is None: + logger.info("inbound message %s matched no Relay address; discarded", message.message_id) + return {"discarded": "no_matching_address", "message_id": message.message_id} + + blocked = find_blocked_sender(message.sender_addresses) + if blocked is not None: + return file_blocked(message, blocked, bucket=bucket, key=key, size_bytes=len(raw)) + + try: + stored = file_message(message, bucket=bucket, key=key, size_bytes=len(raw), s3=s3, recipient=recipient) + except IntegrityError: + # The same message delivered twice. The old path signalled this with a + # DynamoDB conditional write; the unique constraint says the same thing + # and cannot half-succeed, because it is in the same transaction. + logger.info("inbound message %s was already filed; treating as replay", message.message_id) + return {"message_id": message.message_id, "replay": True} + + if stored.state == InboundMessageState.RECEIVED and settings.INBOUND_EMAIL_EVENTS_TOPIC_ARN: + publish_event(publisher, stored, recipient) + + return { + "message_id": stored.message_id, + "inbound_message_id": stored.id, + "state": stored.state, + "replay": False, + } + + +def match_recipient(recipients): + """The Relay-owned address this message was delivered to. + + The database wins over the environment variable, and that order matters: an + address created in the console has to receive mail without a redeploy, and + an address retired in the console has to stop receiving it. The environment + variable is what the sandbox runs on and what an estate with no addresses + yet falls back to. + """ + addresses = [address.lower() for address in recipients] + + for address in addresses: + local_part, _, domain = address.partition("@") + if not local_part or not domain: continue - try: - event = build_event( - message, - event_id=event_id, - route=route, - matched_recipients=matched_recipients, - raw_bucket=bucket, - raw_key=key, - s3=s3, + # Filtered on the two real columns rather than through the `address` + # property, which is Python-side and not queryable. + row = ( + InboundAddress.objects.filter( + local_part__iexact=local_part, + domain__iexact=domain, + is_active=True, ) - publish_event(publisher, event) - except Exception: - release_claim(dynamodb, event_id) - raise - results.append({"event_id": event_id, "route": route, "idempotent_replay": False}) - return results + .order_by("id") + .first() + ) + if row is not None: + return row + if settings.INBOUND_EMAIL_ROUTES: + for address in addresses: + if address in settings.INBOUND_EMAIL_ROUTES: + local_part, _, domain = address.partition("@") + return InboundAddress( + local_part=local_part or address, + domain=domain, + note="Matched from INBOUND_EMAIL_ROUTES; not a managed address.", + ) -def matching_routes(recipients, configured_routes): - matched = {} - for recipient in recipients: - route = configured_routes.get(recipient.lower()) - if route: - matched.setdefault(route, []).append(recipient) - return matched + return None -def build_event(message, *, event_id, route, matched_recipients, raw_bucket, raw_key, s3): - artifact_prefix = f"{settings.INBOUND_EMAIL_ARTIFACT_PREFIX.rstrip('/')}/{event_id}" - body = { - "text": content_value(message.body_text, "text/plain", artifact_prefix, "body.txt", raw_bucket, s3), - "html": content_value(message.body_html, "text/html", artifact_prefix, "body.html", raw_bucket, s3), - } - attachments = [] - for index, attachment in enumerate(message.attachments): - filename = safe_filename(attachment.filename or f"attachment-{index + 1}") - object_key = f"{artifact_prefix}/attachments/{index + 1:03d}-{filename}" +def find_blocked_sender(sender_addresses): + """The rule blocking this sender, if any. + + An exact address beats a domain rule, so blocking one correspondent at a + shared sending domain does not silence everyone else on it. + """ + addresses = [address.lower() for address in sender_addresses if address] + domains = {address.partition("@")[2] for address in addresses if "@" in address} + + if addresses: + rule = BlockedSender.objects.filter( + scope=BlockedSender.Scope.ADDRESS, value__in=addresses + ).first() + if rule is not None: + return rule + + if domains: + return ( + BlockedSender.objects.filter(scope=BlockedSender.Scope.DOMAIN, value__in=sorted(domains)) + .order_by("id") + .first() + ) + + return None + + +def file_message(message, *, bucket, key, size_bytes, s3, recipient): + # Derived from the message id rather than the row, so a redelivery reuses the + # same artifact keys and does not leave a second copy behind. + artifact_id = hashlib.sha256(message.message_id.encode()).hexdigest()[:32] + body_text_key, body_html_key = store_bodies(message, bucket=bucket, s3=s3, artifact_id=artifact_id) + recipient_address = first_recipient(message.recipient_addresses, recipient) + + with transaction.atomic(): + row = InboundMessage.objects.create( + message_id=message.message_id, + state=InboundMessageState.RECEIVED, + subject=message.subject[:998], + snippet=snippet(message.body_text or message.text.preview), + from_header=message.from_[:998], + sender_address=(message.sender_addresses[0] if message.sender_addresses else "")[:320], + sender_domain=sender_domain(message.sender_addresses), + to_header=message.to[:998], + cc_header=message.cc[:998], + recipient=recipient.local_part[:320], + recipient_domain=recipient.domain[:255], + recipient_address=recipient_address[:320], + inbound_address=recipient if recipient.pk else None, + sent_at=parse_date(message.date), + size_bytes=size_bytes, + attachment_count=len(message.attachments), + raw_bucket=bucket, + raw_key=key, + body_text_key=body_text_key, + body_html_key=body_html_key, + spam_verdict=verdict(message, "x-ses-spam-verdict"), + virus_verdict=verdict(message, "x-ses-virus-verdict"), + spf_verdict=verdict(message, "x-ses-spf-verdict"), + dkim_verdict=verdict(message, "x-ses-dkim-verdict"), + dmarc_verdict=verdict(message, "x-ses-dmarc-verdict"), + ) + return row + + +def file_blocked(message, rule, *, bucket, key, size_bytes): + """Record a discarded message so the block is visible and auditable. + + Deliberately does not store the body. The operator already decided this + sender is not worth reading, and a blocklist that accumulates every pitch + ever received is a spam archive rather than a filter. + """ + with transaction.atomic(): + row = InboundMessage.objects.create( + message_id=message.message_id, + state=InboundMessageState.BLOCKED, + subject=message.subject[:998], + snippet=snippet(message.text.preview, limit=BLOCKED_SNIPPET_CHARS), + from_header=message.from_[:998], + sender_address=(message.sender_addresses[0] if message.sender_addresses else "")[:320], + sender_domain=sender_domain(message.sender_addresses), + to_header=message.to[:998], + cc_header=message.cc[:998], + recipient_address=(message.recipient_addresses[0] if message.recipient_addresses else "")[:320], + size_bytes=size_bytes, + raw_bucket=bucket, + raw_key=key, + blocked_rule=rule, + spam_verdict=verdict(message, "x-ses-spam-verdict"), + ) + return {"message_id": row.message_id, "inbound_message_id": row.id, "state": row.state, "replay": False} + + +def store_bodies(message, *, bucket, s3, artifact_id): + """Keep the body parts in S3 and return their keys. + + The raw MIME is already in the bucket, so these are a convenience copy that + spares the console from re-parsing a whole message to show one part. Both are + written with SSE and neither is written when the part is empty. + + Keyed by artifact_id rather than by row id, because the row does not exist + yet. That also makes a redelivery overwrite its own copies instead of + accumulating a second set. + """ + keys = [] + for value, filename, content_type in ( + (message.body_text, "body.txt", "text/plain"), + (message.body_html, "body.html", "text/html"), + ): + if not value: + keys.append("") + continue + object_key = f"{settings.INBOUND_EMAIL_ARTIFACT_PREFIX.rstrip('/')}/{artifact_id}/{filename}" s3.put_object( - Bucket=raw_bucket, + Bucket=bucket, Key=object_key, - Body=attachment.payload, - ContentType=attachment.content_type, + Body=value.encode(), + ContentType=content_type, ServerSideEncryption="AES256", ) - attachments.append( - { - "filename": attachment.filename, - "content_type": attachment.content_type, - "content_id": attachment.content_id, - "disposition": attachment.disposition, - "size": len(attachment.payload), - "checksum": f"sha256:{hashlib.sha256(attachment.payload).hexdigest()}", - "s3": {"bucket": raw_bucket, "key": object_key}, - } + keys.append(object_key) + return keys[0], keys[1] + + +def load_body_text(row, *, s3=None): + """The stored plain-text body, or "" when there is none. + + HTML is not rendered: an inbound HTML body is untrusted third-party markup, + and the console is an authenticated page that would be a poor place to + execute it. + """ + if not row.body_text_key: + return "" + s3 = s3 or s3_client() + try: + return s3.get_object(Bucket=row.raw_bucket, Key=row.body_text_key)["Body"].read().decode( + "utf-8", errors="replace" ) - return { - "contract": CONTRACT, - "version": VERSION, - "event_id": event_id, - "event_type": "email.received", - "occurred_at": datetime.now(UTC).isoformat(), - "route": route, - "message_id": message.message_id, - "sender": {"header": message.from_, "addresses": message.sender_addresses}, - "recipients": { - "to": message.to, - "cc": message.cc, - "addresses": message.recipient_addresses, - "matched": matched_recipients, - }, - "subject": message.subject, - "date": message.date, - "body": body, - "attachments": attachments, - "raw_mime": {"bucket": raw_bucket, "key": raw_key}, - } + except Exception: + logger.exception("could not read stored body for inbound message %s", row.pk) + return "" -def content_value(value, content_type, prefix, filename, bucket, s3): - encoded = value.encode() - if len(encoded) <= settings.INBOUND_EMAIL_INLINE_BODY_MAX_BYTES: - return {"content_type": content_type, "value": value, "size": len(encoded)} - key = f"{prefix}/{filename}" - s3.put_object( - Bucket=bucket, - Key=key, - Body=encoded, - ContentType=content_type, - ServerSideEncryption="AES256", - ) - return {"content_type": content_type, "size": len(encoded), "s3": {"bucket": bucket, "key": key}} +def sender_domain(sender_addresses): + for address in sender_addresses: + _, _, domain = address.partition("@") + if domain: + return domain[:255] + return "" + +def first_recipient(recipient_addresses, recipient): + for address in recipient_addresses: + if address.lower() == recipient.address.lower(): + return address + return recipient.address -def claim_event(dynamodb, event_id, route, message_id): - if not settings.INBOUND_EMAIL_IDEMPOTENCY_TABLE: - raise ImproperlyConfigured("INBOUND_EMAIL_IDEMPOTENCY_TABLE is required") + +def verdict(message, name): + return str(message.selected_headers.get(name, "")).strip()[:64] + + +def parse_date(value): + if not value: + return None try: - dynamodb.put_item( - TableName=settings.INBOUND_EMAIL_IDEMPOTENCY_TABLE, - Item={ - "event_id": {"S": event_id}, - "route": {"S": route}, - "message_id": {"S": message_id}, - "created_at": {"S": datetime.now(UTC).isoformat()}, - }, - ConditionExpression="attribute_not_exists(event_id)", - ) - except dynamodb.exceptions.ConditionalCheckFailedException: - return False - return True + parsed = parsedate_to_datetime(value) + except (TypeError, ValueError): + return None + if parsed is None: + return None + if parsed.tzinfo is None: + return parsed.replace(tzinfo=UTC) + return parsed -def release_claim(dynamodb, event_id): - dynamodb.delete_item(TableName=settings.INBOUND_EMAIL_IDEMPOTENCY_TABLE, Key={"event_id": {"S": event_id}}) +def snippet(value, *, limit=SNIPPET_CHARS): + return " ".join((value or "").split())[:limit] -def publish_event(publisher, event): - if not settings.INBOUND_EMAIL_EVENTS_TOPIC_ARN: - raise ImproperlyConfigured("INBOUND_EMAIL_EVENTS_TOPIC_ARN is required") +def publish_event(publisher, row, recipient): publisher.publish( TopicArn=settings.INBOUND_EMAIL_EVENTS_TOPIC_ARN, - Message=json.dumps(event, sort_keys=True), + Message=json.dumps( + { + "contract": CONTRACT, + "version": VERSION, + "event_type": "email.received", + "occurred_at": datetime.now(UTC).isoformat(), + "route": recipient.address, + "message_id": row.message_id, + "sender": {"header": row.from_header, "addresses": [row.sender_address]}, + "recipients": {"addresses": [row.recipient_address]}, + "subject": row.subject, + "inbound_message_id": row.id, + "raw_mime": {"bucket": row.raw_bucket, "key": row.raw_key}, + }, + sort_keys=True, + ), MessageAttributes={ "contract": {"DataType": "String", "StringValue": CONTRACT}, - "route": {"DataType": "String", "StringValue": event["route"]}, + "route": {"DataType": "String", "StringValue": recipient.address}, }, ) diff --git a/mailing/services/inbound_views.py b/mailing/services/inbound_views.py new file mode 100644 index 0000000..cb99871 --- /dev/null +++ b/mailing/services/inbound_views.py @@ -0,0 +1,201 @@ +"""Query and command helpers for the inbound mailbox. + +Kept out of views.py for the same reason every other view model is: the console +is a client-scoped operator surface, and inbound is not client-scoped, so these +have to be the part of the mailbox that does not care which tenant is selected. +""" + +from django.db.models import Count, Q +from django.utils import timezone + +from mailing.models import ( + BlockedSender, + InboundAddress, + InboundMessage, + InboundMessageState, +) + +# The default view is the unread mail, because that is what an operator opening +# the mailbox is looking for. Read mail is still a click away and is not +# discarded by a filter anywhere. +DEFAULT_STATE = InboundMessageState.RECEIVED + +BLOCK_ORIGIN_BUTTON = "mark_as_spam" +BLOCK_ORIGIN_MANUAL = "manual" + + +def normalize_address(value): + return (value or "").strip().lower() + + +def address_domain(value): + return normalize_address(value).partition("@")[2] + + +def list_messages(*, state=DEFAULT_STATE, query="", address="", page=1, per_page=50): + """The mailbox list. + + `state=None` means every state, which is how the blocked and read mail stay + reachable: a filter that cannot be turned off hides rows permanently. + """ + queryset = InboundMessage.objects.select_related("inbound_address", "blocked_rule") + + if state: + queryset = queryset.filter(state=state) + + query = (query or "").strip() + if query: + queryset = queryset.filter( + Q(subject__icontains=query) + | Q(snippet__icontains=query) + | Q(sender_address__icontains=query) + | Q(from_header__icontains=query) + ) + + address = normalize_address(address) + if address: + queryset = queryset.filter(recipient_address__iexact=address) + + total = queryset.count() + per_page = max(1, min(int(per_page or 50), 200)) + number = max(1, int(page or 1)) + start = (number - 1) * per_page + rows = list(queryset[start : start + per_page]) + pages = max(1, -(-total // per_page)) + + return { + "rows": rows, + "total": total, + "page": number, + "pages": pages, + "per_page": per_page, + "has_previous": number > 1, + "has_next": number < pages, + "previous_page": number - 1, + "next_page": number + 1, + "state": state or "", + "query": query, + "address": address, + "counts": message_counts(), + "sender_count": queryset.exclude(sender_address="").values("sender_address").distinct().count(), + } + + +def message_counts(): + counts = {InboundMessageState.RECEIVED: 0, InboundMessageState.READ: 0, InboundMessageState.BLOCKED: 0} + for state, total in InboundMessage.objects.values("state").annotate(total=Count("id")): + counts[state] = total + counts["all"] = sum(counts[key] for key in (InboundMessageState.RECEIVED, InboundMessageState.READ, InboundMessageState.BLOCKED)) + return counts + + +def unread_sender_count(): + """Distinct senders still waiting to be read. + + This is the number that matters for the "how much junk is waiting" question, + and it is deliberately not the message count: one sender pitching forty times + is one sender. + """ + return ( + InboundMessage.objects.filter(state=InboundMessageState.RECEIVED) + .exclude(sender_address="") + .values("sender_address") + .distinct() + .count() + ) + + +def get_message(message_id): + return InboundMessage.objects.select_related("inbound_address", "blocked_rule").filter(pk=message_id).first() + + +def mark_read(message): + if message.state == InboundMessageState.RECEIVED: + message.state = InboundMessageState.READ + message.read_at = timezone.now() + message.save(update_fields=["state", "read_at", "updated_at"]) + return message + + +def list_addresses(*, include_inactive=True): + queryset = InboundAddress.objects.annotate(message_count=Count("messages")) + if not include_inactive: + queryset = queryset.filter(is_active=True) + return queryset.order_by("domain", "local_part") + + +def address_summary(): + return { + "addresses": list_addresses(), + "active_count": InboundAddress.objects.filter(is_active=True).count(), + "retired_count": InboundAddress.objects.filter(is_active=False).count(), + "total_count": InboundAddress.objects.count(), + } + + +def list_blocked_senders(): + return ( + BlockedSender.objects.select_related("origin_message") + .annotate(blocked_message_count=Count("blocked_messages")) + .order_by("-created_at", "-id") + ) + + +def block_sender(*, message, scope, value, reason="", origin=BLOCK_ORIGIN_BUTTON, actor=None): + """Block a sender, or return the rule that already blocks them. + + Returns `(rule, created)`. The caller needs to know whether it created the + rule so the UI can say "already blocked" instead of pretending it just + stopped something, and because re-blocking the same sender from a second + message is a no-op an operator should see rather than a silent duplicate. + """ + scope = BlockedSender.Scope(scope) + value = normalize_address(value) + if not value: + return None, False + if scope == BlockedSender.Scope.DOMAIN and "@" in value: + # Only an address is split. A value that is already a bare domain has no + # "@", and splitting it anyway yields an empty string -- which would + # silently block nothing. + value = address_domain(value) + if not value: + return None, False + + rule, created = BlockedSender.objects.get_or_create( + scope=scope, + value=value, + defaults={ + "reason": reason or (f"Marked as spam from {message.subject[:120]}" if message else ""), + "origin": origin, + "origin_message": message, + }, + ) + return rule, created + + +def unblock_sender(rule): + rule.delete() + + +def backfill_from_routes(routes): + """Create addresses for every route in INBOUND_EMAIL_ROUTES. + + This is the migration path for a domain that was already receiving mail + through the environment variable: without it, the first messages after a + deploy would find no managed address and be discarded as unmatched. Routes + that already exist are left alone, so retiring an address is not undone by + running this again. + """ + created = [] + for address in routes or {}: + local_part, _, domain = normalize_address(address).partition("@") + if not local_part or not domain: + continue + row, was_created = InboundAddress.objects.get_or_create( + local_part=local_part, + domain=domain, + defaults={"note": "Backfilled from INBOUND_EMAIL_ROUTES."}, + ) + if was_created: + created.append(row) + return created diff --git a/mailing/tests/test_inbound_email.py b/mailing/tests/test_inbound_email.py index fc236a5..227b5d2 100644 --- a/mailing/tests/test_inbound_email.py +++ b/mailing/tests/test_inbound_email.py @@ -1,45 +1,44 @@ +"""Inbound mail is filed into the mailbox, and a blocklist stops it first. + +The service this replaced parsed each message, published an `email.received` +event and kept nothing. These tests cover what replaced that: a message becomes +a row, a message from a blocked sender becomes a different row, and neither +happens twice. Every assertion here fails if the corresponding behaviour breaks, +which is the standard this repository holds its tests to. + +The AWS surface is faked with keyword injection rather than stubbed, so the test +never needs a network or credentials. There is no DynamoDB any more: idempotency +is a unique constraint on the message id, so there is nothing to fake. +""" + import io import json from email.message import EmailMessage -from types import SimpleNamespace import pytest from django.test import override_settings from mailing.inbound_mime import address_values, parse_mime +from mailing.models import BlockedSender, InboundAddress, InboundMessage, InboundMessageState from mailing.services.inbound_email import process_inbound_s3_notification - -class ConditionalCheckFailedException(Exception): - pass - - -class FakeDynamoDB: - exceptions = SimpleNamespace(ConditionalCheckFailedException=ConditionalCheckFailedException) - - def __init__(self): - self.items = {} - - def put_item(self, TableName, Item, ConditionExpression): - key = Item["event_id"]["S"] - if key in self.items: - raise ConditionalCheckFailedException - self.items[key] = Item - - def delete_item(self, TableName, Key): - self.items.pop(Key["event_id"]["S"], None) +pytestmark = pytest.mark.django_db class FakeS3: - def __init__(self, raw): + def __init__(self, raw=None): self.raw = raw self.puts = [] + self.objects = {} def get_object(self, Bucket, Key): + if Key in self.objects: + return {"Body": io.BytesIO(self.objects[Key])} return {"Body": io.BytesIO(self.raw)} - def put_object(self, **kwargs): - self.puts.append(kwargs) + def put_object(self, *, Bucket, Key, Body, **kwargs): + self.puts.append({"Bucket": Bucket, "Key": Key, "Body": Body, **kwargs}) + self.objects[Key] = Body class FakePublisher: @@ -53,35 +52,56 @@ def publish(self, **kwargs): self.messages.append(kwargs) -def raw_message(*, message_id=""): +def raw_message( + *, + message_id="", + sender="Billing ", + to="Invoice , TODO ", + subject="Receipt for July", + spam_verdict=None, +): message = EmailMessage() - message["From"] = "Billing " - message["To"] = "Invoice , TODO " + message["From"] = sender + message["To"] = to message["Cc"] = "Copy " - message["Subject"] = "Receipt for July" + message["Subject"] = subject message["Date"] = "Sun, 12 Jul 2026 09:30:00 +0200" if message_id is not None: message["Message-ID"] = message_id + if spam_verdict is not None: + message["X-SES-Spam-Verdict"] = spam_verdict message.set_content("Plain receipt body") message.add_alternative("HTML receipt body", subtype="html") message.add_attachment(b"pdf bytes", maintype="application", subtype="pdf", filename="receipt July.pdf") return message.as_bytes() -def s3_event(): +def s3_event(key="raw%2Fmessage-123", bucket="inbound-private"): return { "Records": [ { "eventSource": "aws:s3", - "s3": { - "bucket": {"name": "inbound-private"}, - "object": {"key": "raw%2Fmessage-123"}, - }, + "s3": {"bucket": {"name": bucket}, "object": {"key": key}}, } ] } +def deliver(**kwargs): + """Run the worker over one S3 record and return that record's result. + + The worker returns a list, one entry per record, because a single + notification can carry several. Every test here sets up exactly one. + """ + results = process_inbound_s3_notification(s3_event(**kwargs.pop("event", {})), **kwargs) + assert len(results) == 1 + return results[0] + + +def managed_address(local_part="invoice", domain="mailer.test"): + return InboundAddress.objects.create(local_part=local_part, domain=domain) + + def test_parser_extracts_alternatives_addresses_and_attachment(): parsed = parse_mime(raw_message()) @@ -94,72 +114,305 @@ def test_parser_extracts_alternatives_addresses_and_attachment(): assert parsed.attachments[0].payload == b"pdf bytes" +def test_address_values_ignores_display_names(): + assert address_values('"Doe, Jane" ') == ["jane@example.com"] + + +@override_settings(INBOUND_EMAIL_ROUTES={}, INBOUND_EMAIL_ARTIFACT_PREFIX="processed/") +def test_message_is_filed_with_headers_bodies_and_verdicts(): + managed_address() + s3 = FakeS3(raw_message(spam_verdict="Yes")) + + result = deliver(s3=s3) + + row = InboundMessage.objects.get() + assert result["state"] == InboundMessageState.RECEIVED + assert row.subject == "Receipt for July" + assert row.sender_address == "billing@example.com" + assert row.sender_domain == "example.com" + assert row.recipient_address == "invoice@mailer.test" + assert row.inbound_address.local_part == "invoice" + assert row.attachment_count == 1 + assert row.size_bytes == len(s3.raw) + assert row.is_spam is True + assert row.snippet.startswith("Plain receipt body") + # The claimed send date is parsed, not the receipt date. + assert row.sent_at.year == 2026 + assert row.sent_at.month == 7 + # Bodies are stored out of band, and the raw MIME is only pointed at. + assert row.raw_key == "raw/message-123" + assert row.body_text_key.endswith("body.txt") + assert row.body_html_key.endswith("body.html") + assert row.raw_bucket == "inbound-private" + stored_keys = {put["Key"] for put in s3.puts} + assert row.body_text_key in stored_keys + assert row.body_html_key in stored_keys + assert all(put["ServerSideEncryption"] == "AES256" for put in s3.puts) + + +@override_settings(INBOUND_EMAIL_ROUTES={}, INBOUND_EMAIL_ARTIFACT_PREFIX="processed/") +def test_bodies_of_different_messages_do_not_overwrite_each_other(): + """The artifact key is per-message, not a single shared name. + + A shared key would make the second message's body replace the first's, and + the failure would look like a rendering bug rather than a lost message. + """ + address = managed_address() + first = deliver(event={"key": "raw%2Fone"}, s3=FakeS3(raw_message(message_id=""))) + second = deliver(event={"key": "raw%2Ftwo"}, s3=FakeS3(raw_message(message_id=""))) + + one = InboundMessage.objects.get(pk=first["inbound_message_id"]) + two = InboundMessage.objects.get(pk=second["inbound_message_id"]) + assert one.body_text_key != two.body_text_key + assert one.inbound_address_id == two.inbound_address_id == address.pk + + +@override_settings(INBOUND_EMAIL_ROUTES={}, INBOUND_EMAIL_ARTIFACT_PREFIX="processed/") +def test_redelivery_of_the_same_message_is_a_replay_not_a_second_row(): + managed_address() + s3 = FakeS3(raw_message()) + + deliver(s3=s3) + second = deliver(s3=s3) + + assert InboundMessage.objects.count() == 1 + assert second["replay"] is True + + +@override_settings(INBOUND_EMAIL_ROUTES={}, INBOUND_EMAIL_ARTIFACT_PREFIX="processed/") +def test_address_match_is_case_insensitive(): + managed_address(local_part="Invoice", domain="Mailer.Test") + address = InboundAddress.objects.get() + + deliver(s3=FakeS3(raw_message())) + + row = InboundMessage.objects.get() + assert row.inbound_address_id == address.pk + assert row.recipient_address == "invoice@mailer.test" + + +@override_settings(INBOUND_EMAIL_ROUTES={}, INBOUND_EMAIL_ARTIFACT_PREFIX="processed/") +def test_message_for_an_unknown_address_is_discarded(): + s3 = FakeS3(raw_message()) + + result = deliver(s3=s3) + + assert result["discarded"] == "no_matching_address" + assert not InboundMessage.objects.exists() + # Nothing is written either: an unroutable message is not worth a body copy. + assert s3.puts == [] + + +@override_settings(INBOUND_EMAIL_ROUTES={}, INBOUND_EMAIL_ARTIFACT_PREFIX="processed/") +def test_a_retired_address_stops_receiving(): + address = managed_address() + address.is_active = False + address.save(update_fields=["is_active"]) + + result = deliver(s3=FakeS3(raw_message())) + + assert result["discarded"] == "no_matching_address" + assert not InboundMessage.objects.exists() + + @override_settings( - INBOUND_EMAIL_ROUTES={"invoice@mailer.test": "invoice", "todo@mailer.test": "todo"}, - INBOUND_EMAIL_IDEMPOTENCY_TABLE="inbound-idempotency", - INBOUND_EMAIL_EVENTS_TOPIC_ARN="arn:aws:sns:eu-west-1:123:inbound-events", + INBOUND_EMAIL_ROUTES={"invoice@mailer.test": "invoice"}, INBOUND_EMAIL_ARTIFACT_PREFIX="processed/", - INBOUND_EMAIL_INLINE_BODY_MAX_BYTES=65536, ) -def test_worker_routes_multiple_aliases_persists_attachment_and_is_idempotent(): - s3 = FakeS3(raw_message()) - dynamodb = FakeDynamoDB() - publisher = FakePublisher() +def test_environment_routes_still_work_when_no_managed_address_exists(): + """The sandbox runs on INBOUND_EMAIL_ROUTES and must keep receiving. - first = process_inbound_s3_notification(s3_event(), s3=s3, dynamodb=dynamodb, publisher=publisher) - second = process_inbound_s3_notification(s3_event(), s3=s3, dynamodb=dynamodb, publisher=publisher) + The database wins when it has an address, and the environment is the + fallback, so an estate that has created no addresses is not silently dark. + """ + result = deliver(s3=FakeS3(raw_message())) - assert [item["route"] for item in first] == ["invoice", "todo"] - assert all(item["idempotent_replay"] is False for item in first) - assert all(item["idempotent_replay"] is True for item in second) - assert len(publisher.messages) == 2 - events = [json.loads(item["Message"]) for item in publisher.messages] - assert {event["route"] for event in events} == {"invoice", "todo"} - assert all(event["contract"] == "inbound-email" and event["version"] == 1 for event in events) - assert all(event["raw_mime"] == {"bucket": "inbound-private", "key": "raw/message-123"} for event in events) - assert all(event["attachments"][0]["s3"]["bucket"] == "inbound-private" for event in events) - assert len(s3.puts) == 2 + row = InboundMessage.objects.get() + assert result["state"] == InboundMessageState.RECEIVED + assert row.recipient_address == "invoice@mailer.test" + # Not a managed address, so the row is not filed under one. + assert row.inbound_address_id is None @override_settings( INBOUND_EMAIL_ROUTES={"invoice@mailer.test": "invoice"}, - INBOUND_EMAIL_IDEMPOTENCY_TABLE="inbound-idempotency", - INBOUND_EMAIL_EVENTS_TOPIC_ARN="arn:aws:sns:eu-west-1:123:inbound-events", INBOUND_EMAIL_ARTIFACT_PREFIX="processed/", - INBOUND_EMAIL_INLINE_BODY_MAX_BYTES=4, ) -def test_worker_stores_large_bodies_and_releases_claim_after_publish_failure(): +def test_a_managed_address_takes_precedence_over_the_environment_route(): + managed_address() + InboundMessage.objects.create( + message_id="", + recipient_address="invoice@mailer.test", + inbound_address=InboundAddress.objects.get(), + ) + + deliver(s3=FakeS3(raw_message())) + + assert InboundMessage.objects.get(message_id="").inbound_address_id is not None + + +@override_settings(INBOUND_EMAIL_ROUTES={}, INBOUND_EMAIL_ARTIFACT_PREFIX="processed/") +def test_blocked_sender_is_discarded_and_recorded_without_a_body(): + """Blocking stops storage, not just display. + + A blocklist that still files every message is a filter an operator has to + re-apply by hand, which is the thing it was meant to replace. + """ + managed_address() + rule = BlockedSender.objects.create(scope=BlockedSender.Scope.ADDRESS, value="billing@example.com") s3 = FakeS3(raw_message()) - dynamodb = FakeDynamoDB() - with pytest.raises(RuntimeError, match="publish failed"): - process_inbound_s3_notification(s3_event(), s3=s3, dynamodb=dynamodb, publisher=FakePublisher(fail=True)) + result = deliver(s3=s3) + + row = InboundMessage.objects.get() + assert result["state"] == InboundMessageState.BLOCKED + assert row.state == InboundMessageState.BLOCKED + assert row.blocked_rule_id == rule.pk + assert row.sender_address == "billing@example.com" + assert row.body_text_key == "" + assert row.raw_key == "raw/message-123" + assert s3.puts == [] + + +@override_settings(INBOUND_EMAIL_ROUTES={}, INBOUND_EMAIL_ARTIFACT_PREFIX="processed/") +def test_a_domain_block_catches_every_sender_on_it(): + managed_address() + BlockedSender.objects.create(scope=BlockedSender.Scope.DOMAIN, value="example.com") + + result = deliver(s3=FakeS3(raw_message())) + + assert result["state"] == InboundMessageState.BLOCKED + assert InboundMessage.objects.get().state == InboundMessageState.BLOCKED - assert dynamodb.items == {} + +@override_settings(INBOUND_EMAIL_ROUTES={}, INBOUND_EMAIL_ARTIFACT_PREFIX="processed/") +def test_an_address_block_wins_over_a_domain_block_for_that_sender(): + """One correspondent at a shared sending domain can be blocked alone. + + Both rules match other@example.com, and the narrower one is the one an + operator means when they clicked the address and not the domain. + """ + managed_address() + BlockedSender.objects.create(scope=BlockedSender.Scope.DOMAIN, value="example.com") + address_rule = BlockedSender.objects.create(scope=BlockedSender.Scope.ADDRESS, value="other@example.com") + + deliver( + event={"key": "raw%2Fother"}, + s3=FakeS3(raw_message(message_id="", sender="Other ")), + ) + + blocked_row = InboundMessage.objects.get(message_id="") + assert blocked_row.state == InboundMessageState.BLOCKED + assert blocked_row.blocked_rule_id == address_rule.pk + + + +@override_settings(INBOUND_EMAIL_ROUTES={}, INBOUND_EMAIL_ARTIFACT_PREFIX="processed/") +def test_unblocking_lets_the_sender_through_again(): + managed_address() + rule = BlockedSender.objects.create(scope=BlockedSender.Scope.ADDRESS, value="billing@example.com") + process_inbound_s3_notification(s3_event(), s3=FakeS3(raw_message())) + rule.delete() + + deliver(event={"key": "raw%2Fsecond"}, s3=FakeS3(raw_message(message_id=""))) + + assert InboundMessage.objects.get(message_id="").state == InboundMessageState.RECEIVED + + +@override_settings( + INBOUND_EMAIL_ROUTES={}, + INBOUND_EMAIL_ARTIFACT_PREFIX="processed/", + INBOUND_EMAIL_EVENTS_TOPIC_ARN="arn:aws:sns:eu-west-1:123:inbound-events", +) +def test_event_is_published_when_a_topic_is_configured(): + managed_address() publisher = FakePublisher() - process_inbound_s3_notification(s3_event(), s3=s3, dynamodb=dynamodb, publisher=publisher) - event = json.loads(publisher.messages[0]["Message"]) - assert event["body"]["text"]["s3"]["key"].endswith("/body.txt") - assert event["body"]["html"]["s3"]["key"].endswith("/body.html") + + deliver(s3=FakeS3(raw_message()), publisher=publisher) + + assert len(publisher.messages) == 1 + body = json.loads(publisher.messages[0]["Message"]) + assert body["contract"] == "inbound-email" + assert body["event_type"] == "email.received" + assert body["route"] == "invoice@mailer.test" + assert body["inbound_message_id"] == InboundMessage.objects.get().pk + assert body["raw_mime"] == {"bucket": "inbound-private", "key": "raw/message-123"} @override_settings( - INBOUND_EMAIL_ROUTES={"invoice@mailer.test": "invoice"}, - INBOUND_EMAIL_IDEMPOTENCY_TABLE="inbound-idempotency", + INBOUND_EMAIL_ROUTES={}, + INBOUND_EMAIL_ARTIFACT_PREFIX="processed/", INBOUND_EMAIL_EVENTS_TOPIC_ARN="arn:aws:sns:eu-west-1:123:inbound-events", ) -def test_worker_rejects_message_without_message_id(): - with pytest.raises(ValueError, match="missing Message-ID"): - process_inbound_s3_notification( - s3_event(), s3=FakeS3(raw_message(message_id=None)), dynamodb=FakeDynamoDB(), publisher=FakePublisher() - ) +def test_a_blocked_message_is_not_announced(): + managed_address() + BlockedSender.objects.create(scope=BlockedSender.Scope.ADDRESS, value="billing@example.com") + publisher = FakePublisher() + deliver(s3=FakeS3(raw_message()), publisher=publisher) + + assert publisher.messages == [] + + +@override_settings(INBOUND_EMAIL_ROUTES={}, INBOUND_EMAIL_ARTIFACT_PREFIX="processed/") +def test_nothing_is_published_when_no_topic_is_configured(): + """No topic means no publish, not a failed task. + + A Relay that owns a mailbox does not need a second copy announced over SNS, + and requiring one would make the mailbox depend on a consumer that may not + exist. + """ + managed_address() + publisher = FakePublisher() + + deliver(s3=FakeS3(raw_message()), publisher=publisher) + + assert publisher.messages == [] + assert InboundMessage.objects.count() == 1 + + +@override_settings(INBOUND_EMAIL_ROUTES={}, INBOUND_EMAIL_ARTIFACT_PREFIX="processed/") +def test_a_long_message_id_is_truncated_rather_than_failing_the_write(): + """The unique index is a btree; a 998-character id would exceed its limit. + + Truncating keeps the dedupe guarantee and moves the failure from an insert + error to a shortened value. + """ + managed_address() + long_id = "<" + ("a" * 1200) + "@example.com>" + + deliver(s3=FakeS3(raw_message(message_id=long_id))) + + row = InboundMessage.objects.get() + assert len(row.message_id) == 512 + + +@override_settings(INBOUND_EMAIL_ROUTES={}, INBOUND_EMAIL_ARTIFACT_PREFIX="processed/") +@pytest.mark.parametrize( + "raw,error", + [(b"", "empty MIME"), (b"not a MIME message", "no headers")], +) +def test_garbage_is_rejected_before_anything_is_stored(raw, error): + managed_address() -@pytest.mark.parametrize("raw,error", [(b"", "empty MIME"), (b"not a MIME message", "no headers")]) -def test_parser_rejects_malformed_mime(raw, error): with pytest.raises(ValueError, match=error): - parse_mime(raw) + deliver(s3=FakeS3(raw)) + assert not InboundMessage.objects.exists() -def test_address_values_ignores_empty_optional_headers(): - assert address_values("invoice@example.com", "", None) == ["invoice@example.com"] + +@override_settings(INBOUND_EMAIL_ROUTES={}, INBOUND_EMAIL_ARTIFACT_PREFIX="processed/") +def test_a_message_without_a_message_id_is_rejected(): + managed_address() + + with pytest.raises(ValueError, match="missing Message-ID"): + deliver(s3=FakeS3(raw_message(message_id=None))) + + assert not InboundMessage.objects.exists() + + +def test_non_s3_records_are_refused(): + with pytest.raises(ValueError, match="only S3 event records"): + process_inbound_s3_notification( + {"Records": [{"eventSource": "aws:sqs"}]}, s3=FakeS3(raw_message()) + ) diff --git a/mailing/tests/test_inbound_ui.py b/mailing/tests/test_inbound_ui.py new file mode 100644 index 0000000..591133c --- /dev/null +++ b/mailing/tests/test_inbound_ui.py @@ -0,0 +1,387 @@ +"""The operator side of the inbound mailbox: the list, the detail page, and the +button that blocks a sender. + +Each test here drives a URL and asserts on what an operator would see or the +row that resulted. The storage behaviour is covered in test_inbound_email.py; +what matters here is that the console can reach it, that the button works, and +that a non-staff user cannot reach any of it. +""" + +import pytest +from django.contrib.auth import get_user_model +from django.urls import reverse + +from mailing.models import BlockedSender, InboundAddress, InboundMessage, InboundMessageState + +pytestmark = pytest.mark.django_db + + +@pytest.fixture +def operator(): + return get_user_model().objects.create_user("operator", "operator@example.com", "password", is_staff=True) + + +@pytest.fixture +def outsider(): + return get_user_model().objects.create_user("outsider", "outsider@example.com", "password") + + +@pytest.fixture +def address(): + return InboundAddress.objects.create(local_part="support", domain="aishippinglabs.com", note="Support inbox") + + +def received(**kwargs): + defaults = { + "message_id": "", + "state": InboundMessageState.RECEIVED, + "subject": "Question about your pricing", + "snippet": "Hello, I saw your site and wanted to ask about pricing.", + "from_header": "Jane ", + "sender_address": "jane@prospect.example", + "sender_domain": "prospect.example", + "recipient": "support", + "recipient_domain": "aishippinglabs.com", + "recipient_address": "support@aishippinglabs.com", + "raw_bucket": "relay-production-inbound-mail", + "raw_key": "raw/abc", + "size_bytes": 4096, + } + defaults.update(kwargs) + return InboundMessage.objects.create(**defaults) + + +def test_the_mailbox_requires_a_staff_login(client, outsider, address): + received() + response = client.get(reverse("mailing:inbound_list")) + + assert response.status_code == 302 + assert b"jane@prospect.example" not in response.content + + +def test_every_inbound_page_is_closed_to_non_staff(client, outsider, address): + client.force_login(outsider) + message = received() + + for name, args in ( + ("mailing:inbound_list", []), + ("mailing:inbound_message_detail", [message.pk]), + ("mailing:inbound_address_list", []), + ("mailing:inbound_address_create", []), + ): + assert client.get(reverse(name, args=args)).status_code == 302, name + + +def test_the_list_shows_unread_mail_with_the_sender_and_subject(client, operator, address): + received() + client.force_login(operator) + + response = client.get(reverse("mailing:inbound_list")) + + assert response.status_code == 200 + assert b"jane@prospect.example" in response.content + assert b"Question about your pricing" in response.content + assert b"support@aishippinglabs.com" in response.content + + +def test_the_list_defaults_to_unread_and_read_mail_is_reachable(client, operator, address): + read = received(message_id="", state=InboundMessageState.READ) + client.force_login(operator) + + unread = client.get(reverse("mailing:inbound_list")) + read_page = client.get(reverse("mailing:inbound_list"), {"state": "read"}) + + assert read.subject.encode() not in unread.content + assert read.subject.encode() in read_page.content + + +def test_search_matches_the_sender_the_subject_and_the_body(client, operator, address): + received() + client.force_login(operator) + + by_sender = client.get(reverse("mailing:inbound_list"), {"q": "prospect.example"}) + by_body = client.get(reverse("mailing:inbound_list"), {"q": "wanted to ask"}) + by_nothing = client.get(reverse("mailing:inbound_list"), {"q": "cryptocurrency"}) + + assert by_sender.status_code == 200 + assert b"Question about your pricing" in by_sender.content + assert b"Question about your pricing" in by_body.content + assert b"Question about your pricing" not in by_nothing.content + + +def test_an_empty_mailbox_says_so_without_claiming_nothing_arrived(client, operator): + client.force_login(operator) + + response = client.get(reverse("mailing:inbound_list")) + + assert response.status_code == 200 + assert b"No unread mail" in response.content + + +def test_a_filtered_empty_result_does_not_claim_nothing_arrived(client, operator, address): + received() + client.force_login(operator) + + response = client.get(reverse("mailing:inbound_list"), {"q": "nothing matches this"}) + + assert b"No messages match this filter" in response.content + assert b"No unread mail" not in response.content + + +def test_opening_a_message_marks_it_read(client, operator, address): + message = received() + client.force_login(operator) + + client.get(reverse("mailing:inbound_message_detail", args=[message.pk])) + message.refresh_from_db() + + assert message.state == InboundMessageState.READ + assert message.read_at is not None + + +def test_mark_as_spam_blocks_the_sender_and_stops_future_mail(client, operator, address): + """The button's whole contract: this sender is not filed again. + + The row the operator was looking at is kept and marked, so the message they + were reading does not vanish from under them, and the block applies to the + next message rather than to the one already read. + """ + message = received() + client.force_login(operator) + + response = client.post( + reverse("mailing:inbound_message_detail", args=[message.pk]), + {"action": "mark_as_spam", "scope": "address"}, + follow=True, + ) + + assert response.status_code == 200 + rule = BlockedSender.objects.get() + assert rule.scope == BlockedSender.Scope.ADDRESS + assert rule.value == "jane@prospect.example" + assert rule.origin_message_id == message.pk + assert rule.origin == "mark_as_spam" + message.refresh_from_db() + assert message.state == InboundMessageState.READ + assert message.blocked_rule_id == rule.pk + assert b"Blocked sender jane@prospect.example" in response.content + + +def test_mark_as_spam_can_block_the_whole_domain(client, operator, address): + message = received() + client.force_login(operator) + + client.post( + reverse("mailing:inbound_message_detail", args=[message.pk]), + {"action": "mark_as_spam", "scope": "domain"}, + ) + + rule = BlockedSender.objects.get() + assert rule.scope == BlockedSender.Scope.DOMAIN + assert rule.value == "prospect.example" + + +def test_blocking_a_second_message_from_the_same_sender_does_not_duplicate(client, operator, address): + first = received() + second = received(message_id="") + client.force_login(operator) + + client.post( + reverse("mailing:inbound_message_detail", args=[first.pk]), + {"action": "mark_as_spam", "scope": "address"}, + ) + response = client.post( + reverse("mailing:inbound_message_detail", args=[second.pk]), + {"action": "mark_as_spam", "scope": "address"}, + follow=True, + ) + + assert BlockedSender.objects.count() == 1 + # The operator is told it was already blocked rather than shown a silent no-op. + assert b"was already blocked" in response.content + + +def test_a_blocked_message_offers_unblocking(client, operator, address): + rule = BlockedSender.objects.create(scope=BlockedSender.Scope.ADDRESS, value="jane@prospect.example") + message = received(state=InboundMessageState.BLOCKED, blocked_rule=rule) + client.force_login(operator) + + response = client.get(reverse("mailing:inbound_message_detail", args=[message.pk])) + + assert b"Unblock this sender" in response.content + assert b"will keep future mail" not in response.content + + +def test_a_blocked_message_explains_that_its_body_was_not_kept(client, operator, address): + rule = BlockedSender.objects.create(scope=BlockedSender.Scope.ADDRESS, value="jane@prospect.example") + message = received( + state=InboundMessageState.BLOCKED, + blocked_rule=rule, + body_text_key="processed/abc/body.txt", + ) + client.force_login(operator) + + response = client.get(reverse("mailing:inbound_message_detail", args=[message.pk])) + + assert b"the body was not kept" in response.content + # A blocked message must not render a body even if one is somehow present. + assert b"wanted to ask about pricing" not in response.content + + +def test_unblocking_from_the_message_lets_the_sender_through(client, operator, address): + rule = BlockedSender.objects.create(scope=BlockedSender.Scope.ADDRESS, value="jane@prospect.example") + message = received(state=InboundMessageState.BLOCKED, blocked_rule=rule) + client.force_login(operator) + + client.post(reverse("mailing:inbound_message_detail", args=[message.pk]), {"action": "unblock"}) + + assert not BlockedSender.objects.exists() + + +def test_a_message_with_no_sender_address_cannot_be_blocked(client, operator, address): + message = received(sender_address="", sender_domain="", from_header="") + client.force_login(operator) + + response = client.post( + reverse("mailing:inbound_message_detail", args=[message.pk]), + {"action": "mark_as_spam", "scope": "address"}, + follow=True, + ) + + assert not BlockedSender.objects.exists() + assert b"no sender address to block" in response.content + + +def test_the_address_page_lists_addresses_and_their_message_counts(client, operator, address): + received() + client.force_login(operator) + + response = client.get(reverse("mailing:inbound_address_list")) + + assert response.status_code == 200 + assert b"support@aishippinglabs.com" in response.content + assert b"Support inbox" in response.content + + +def test_creating_an_address_is_a_row_not_a_deploy(client, operator): + """The whole point: a new address needs no Terraform and no redeploy.""" + client.force_login(operator) + + response = client.post( + reverse("mailing:inbound_address_create"), + {"local_part": "sales", "domain": "aishippinglabs.com", "note": "Inbound leads"}, + follow=True, + ) + + assert response.status_code == 200 + created = InboundAddress.objects.get() + assert created.address == "sales@aishippinglabs.com" + assert created.local_part == "sales" + assert created.is_active is True + assert b"sales@aishippinglabs.com" in response.content + + +def test_creating_an_address_normalises_case(client, operator): + client.force_login(operator) + + client.post( + reverse("mailing:inbound_address_create"), + {"local_part": "Sales", "domain": "AIShippingLabs.COM"}, + ) + + created = InboundAddress.objects.get() + assert created.address == "sales@aishippinglabs.com" + + +def test_creating_a_duplicate_address_is_reported_not_silently_ignored(client, operator, address): + client.force_login(operator) + + response = client.post( + reverse("mailing:inbound_address_create"), + {"local_part": "support", "domain": "aishippinglabs.com"}, + follow=True, + ) + + assert InboundAddress.objects.count() == 1 + assert b"already exists" in response.content + + +def test_an_address_with_an_at_sign_in_the_local_part_is_rejected(client, operator): + client.force_login(operator) + + response = client.post( + reverse("mailing:inbound_address_create"), + {"local_part": "sales@team", "domain": "aishippinglabs.com"}, + follow=True, + ) + + assert not InboundAddress.objects.exists() + assert b"no @" in response.content + + +def test_a_missing_domain_is_rejected(client, operator): + client.force_login(operator) + + response = client.post( + reverse("mailing:inbound_address_create"), + {"local_part": "sales", "domain": ""}, + follow=True, + ) + + assert not InboundAddress.objects.exists() + assert b"Enter a domain" in response.content + + +def test_retiring_an_address_stops_receiving_but_keeps_its_mail(client, operator, address): + received() + client.force_login(operator) + + client.post(reverse("mailing:inbound_address_archive", args=[address.pk])) + address.refresh_from_db() + + assert address.is_active is False + # The message that arrived before the retirement is not collateral damage. + assert InboundMessage.objects.count() == 1 + + +def test_a_retired_address_can_be_reactivated(client, operator, address): + address.is_active = False + address.save(update_fields=["is_active"]) + client.force_login(operator) + + client.post(reverse("mailing:inbound_address_archive", args=[address.pk])) + address.refresh_from_db() + + assert address.is_active is True + + +def test_a_blocked_sender_can_be_unblocked_from_the_address_page(client, operator, address): + rule = BlockedSender.objects.create(scope=BlockedSender.Scope.ADDRESS, value="jane@prospect.example") + client.force_login(operator) + + client.post(reverse("mailing:blocked_sender_delete", args=[rule.pk])) + + assert not BlockedSender.objects.exists() + + +def test_the_ses_spam_verdict_is_shown_as_a_signal_not_as_a_verdict(client, operator, address): + """A verdict is a claim by the sending domain; the page has to say so. + + Showing "spam" next to a message with no explanation invites an operator to + act on SES's opinion as though it were a finding. + """ + received(spam_verdict="Yes") + client.force_login(operator) + + response = client.get(reverse("mailing:inbound_list")) + + assert b"SES: spam" in response.content + + +def test_the_nav_reaches_the_mailbox_and_the_addresses(client, operator, address): + client.force_login(operator) + + html = client.get(reverse("mailing:dashboard")).content.decode() + + assert reverse("mailing:inbound_list") in html + assert reverse("mailing:inbound_address_list") in html diff --git a/mailing/urls.py b/mailing/urls.py index b971086..c0eb5a1 100644 --- a/mailing/urls.py +++ b/mailing/urls.py @@ -69,6 +69,24 @@ path("api-docs/", views.api_docs, name="api_docs"), path("api-docs/openapi.json", views.api_docs_json, name="api_docs_json"), path("api/workers/status", views.api_worker_status, name="api_worker_status"), + path("inbound/", views.inbound_list, name="inbound_list"), + path( + "inbound/messages//", + views.inbound_message_detail, + name="inbound_message_detail", + ), + path("inbound/addresses/", views.inbound_address_list, name="inbound_address_list"), + path("inbound/addresses/new/", views.inbound_address_create, name="inbound_address_create"), + path( + "inbound/addresses//archive/", + views.inbound_address_archive, + name="inbound_address_archive", + ), + path( + "inbound/blocked//delete/", + views.blocked_sender_delete, + name="blocked_sender_delete", + ), path("templates/", views.template_catalog, name="template_catalog"), path("templates//", views.template_detail, name="template_detail"), path("transactional/queue/", views.transactional_queue, name="transactional_queue"), diff --git a/mailing/views.py b/mailing/views.py index cf94290..0766a0e 100644 --- a/mailing/views.py +++ b/mailing/views.py @@ -27,6 +27,7 @@ ) from mailing.models import ( Audience, + BlockedSender, CallbackEndpoint, Campaign, CampaignRecipient, @@ -37,6 +38,8 @@ CmpCallback, EmailEvent, EmailEventType, + InboundAddress, + InboundMessage, MailchimpSync, MailchimpTagMapping, Tag, @@ -95,6 +98,17 @@ export_contacts_csv_for_client, export_contacts_for_client, ) +from mailing.services.inbound_views import ( + BLOCK_ORIGIN_BUTTON, + DEFAULT_STATE, + address_summary, + block_sender, + list_addresses, + list_blocked_senders, + list_messages, + mark_read, + unblock_sender, +) from mailing.services.mailchimp import ( mailchimp_status_payload, reconcile_tag_mappings_for_client, @@ -376,6 +390,166 @@ def transactional_message_detail(request, message_id): ) +@staff_member_required +@require_GET +def inbound_list(request): + """The receiving mailbox. + + Not client-scoped and deliberately so: a receiving address belongs to the + Relay service, not to the tenant selected in the console, and scoping this + to a client would hide mail that arrived for a different one. The address + filter is how an operator narrows it. + """ + state = request.GET.get("state") + if state == "all": + state = None + elif not state: + # No state in the query means the default view, which is unread. It + # cannot be None here, because None means "no filter" and the mailbox + # would open on everything an operator has already dealt with. + state = DEFAULT_STATE + + query = request.GET.get("q", "") + address_filter = request.GET.get("address", "") + context = list_messages( + state=state, + query=query, + address=address_filter, + page=request.GET.get("page", 1), + per_page=request.GET.get("per_page", 50), + ) + # Whether anything narrowed this view. The template cannot infer it from + # `state`, because the default view has a state too -- it is just the one + # an operator did not choose, and "no unread mail" is the honest thing to + # say there where "no messages match" would be a lie about the mailbox. + narrowed = bool(query or address_filter or request.GET.get("state")) + return render( + request, + "mailing/operator/inbound_list.html", + { + **context, + "narrowed": narrowed, + "addresses": list_addresses(), + "blocked_senders": list_blocked_senders()[:25], + }, + ) + + +@staff_member_required +def inbound_message_detail(request, message_id): + message = get_object_or_404(InboundMessage, pk=message_id) + if request.method == "POST": + action = request.POST.get("action", "") + if action == "mark_read": + mark_read(message) + messages.success(request, "Message marked as read.") + elif action == BLOCK_ORIGIN_BUTTON: + scope = request.POST.get("scope", "address") + blocked, created = block_sender( + message=message, + scope=scope, + value=message.sender_address if scope == "address" else message.sender_domain, + origin=BLOCK_ORIGIN_BUTTON, + actor=request.user, + ) + if blocked is None: + messages.error(request, "That message has no sender address to block.") + elif created: + verb = "domain" if scope == "domain" else "sender" + messages.success(request, f"Blocked {verb} {blocked.value}. Future mail is discarded.") + if message.state == "received": + message.state = "read" + message.blocked_rule = blocked + message.save(update_fields=["state", "blocked_rule", "updated_at"]) + else: + messages.info(request, f"{blocked.value} was already blocked.") + elif action == "unblock": + if message.blocked_rule_id: + unblock_sender(message.blocked_rule) + messages.success(request, "Sender unblocked.") + return redirect("mailing:inbound_message_detail", message_id=message.pk) + + body = "" + if message.has_body: + from mailing.services.inbound_email import load_body_text # noqa: PLC0415 - avoids an import cycle + + body = load_body_text(message) + if request.method == "GET" and message.state == "received": + mark_read(message) + + return render( + request, + "mailing/operator/inbound_message_detail.html", + { + "message": message, + "body": body, + "badge": Badge( + message.get_state_display(), + {"blocked": "danger", "read": "neutral"}.get(message.state, "warning"), + ), + "blocked_rule": message.blocked_rule, + }, + ) + + +@staff_member_required +def inbound_address_list(request): + return render( + request, + "mailing/operator/inbound_address_list.html", + { + **address_summary(), + "blocked_rules": list_blocked_senders(), + "routes": sorted(settings.INBOUND_EMAIL_ROUTES), + }, + ) + + +@staff_member_required +@require_http_methods(["GET", "POST"]) +def inbound_address_create(request): + if request.method == "POST": + local_part = request.POST.get("local_part", "").strip().lower() + domain = request.POST.get("domain", "").strip().lower() + note = request.POST.get("note", "").strip() + if not local_part or "@" in local_part: + messages.error(request, "Enter a local part with no @ and no spaces.") + elif not domain or "@" in domain: + messages.error(request, "Enter a domain with no @.") + else: + address, created = InboundAddress.objects.get_or_create( + local_part=local_part, + domain=domain, + defaults={"note": note}, + ) + if created: + messages.success(request, f"{address.address} will receive mail from now on.") + else: + messages.info(request, f"{address.address} already exists.") + return redirect("mailing:inbound_address_list") + return render(request, "mailing/operator/inbound_address_form.html", {"domain_hint": settings.INBOUND_EMAIL_ROUTES}) + + +@staff_member_required +@require_POST +def inbound_address_archive(request, address_id): + address = get_object_or_404(InboundAddress, pk=address_id) + address.is_active = not address.is_active + address.save(update_fields=["is_active", "updated_at"]) + verb = "Retired" if not address.is_active else "Reactivated" + messages.success(request, f"{verb} {address.address}.") + return redirect("mailing:inbound_address_list") + + +@staff_member_required +@require_POST +def blocked_sender_delete(request, blocked_id): + rule = get_object_or_404(BlockedSender, pk=blocked_id) + unblock_sender(rule) + messages.success(request, f"{rule.value} will receive mail again.") + return redirect("mailing:inbound_address_list") + + @staff_member_required def campaign_list(request): active_client = require_active_client(request) diff --git a/relay/dev_auth.py b/relay/dev_auth.py new file mode 100644 index 0000000..954b306 --- /dev/null +++ b/relay/dev_auth.py @@ -0,0 +1,27 @@ +from django.conf import settings +from django.contrib.auth import get_user_model, login + + +class DevAutoLoginMiddleware: + """Pre-authenticate requests as the seeded superuser while DEBUG is on. + + Deployed environments go through the OIDC provider (relay.oidc); this + exists only so a local runserver does not put the Django admin login + between the developer and the operator UI. DEBUG is False for the whole + test suite, so the middleware is dormant there. + """ + + def __init__(self, get_response): + self.get_response = get_response + + def __call__(self, request): + if settings.DEBUG and not request.user.is_authenticated: + user = ( + get_user_model() + .objects.filter(is_active=True, is_superuser=True) + .order_by("pk") + .first() + ) + if user is not None: + login(request, user) + return self.get_response(request) diff --git a/relay/settings.py b/relay/settings.py index a989a6c..5b93442 100644 --- a/relay/settings.py +++ b/relay/settings.py @@ -132,6 +132,10 @@ def float_env(name, *, default): "django.middleware.common.CommonMiddleware", "django.middleware.csrf.CsrfViewMiddleware", "django.contrib.auth.middleware.AuthenticationMiddleware", + # Local convenience only: with DEBUG on, every request is pre-authenticated + # as the seeded superuser instead of hitting the admin login. Deployed + # environments run with DEBUG off and authenticate through relay.oidc. + "relay.dev_auth.DevAutoLoginMiddleware", "django.contrib.messages.middleware.MessageMiddleware", "django.middleware.clickjacking.XFrameOptionsMiddleware", ] @@ -194,6 +198,11 @@ def float_env(name, *, default): {"NAME": "django.contrib.auth.password_validation.NumericPasswordValidator"}, ] +# Where the stock auth views send an already-authenticated visitor (e.g. after +# logging out locally, the admin login page bounces here). The OIDC flow keeps +# its own explicit return_to handling and never reads this. +LOGIN_REDIRECT_URL = "/" + LANGUAGE_CODE = "en-us" TIME_ZONE = "UTC" USE_I18N = True @@ -245,9 +254,11 @@ def float_env(name, *, default): SQS_SANDBOX_SES_WEBHOOKS_QUEUE_URL = os.environ.get("SQS_SANDBOX_SES_WEBHOOKS_QUEUE_URL", "") SQS_SANDBOX_INBOUND_EMAIL_QUEUE_URL = os.environ.get("SQS_SANDBOX_INBOUND_EMAIL_QUEUE_URL", "") INBOUND_EMAIL_EVENTS_TOPIC_ARN = os.environ.get("INBOUND_EMAIL_EVENTS_TOPIC_ARN", "") -INBOUND_EMAIL_IDEMPOTENCY_TABLE = os.environ.get("INBOUND_EMAIL_IDEMPOTENCY_TABLE", "") +# INBOUND_EMAIL_IDEMPOTENCY_TABLE is deliberately not read any more. Idempotency +# is a unique constraint on inbound_messages.message_id, inside the transaction +# that was already happening, so it needs no table and cannot half-succeed. The +# env var is left set in deployed environments until nobody reads it. INBOUND_EMAIL_ARTIFACT_PREFIX = os.environ.get("INBOUND_EMAIL_ARTIFACT_PREFIX", "processed/") -INBOUND_EMAIL_INLINE_BODY_MAX_BYTES = int(os.environ.get("INBOUND_EMAIL_INLINE_BODY_MAX_BYTES", "65536")) INBOUND_EMAIL_ROUTES = { address.strip().lower(): route.strip() for address, separator, route in ( diff --git a/static/mailing/css/app.css b/static/mailing/css/app.css index 5ddb15d..38ed4a7 100644 --- a/static/mailing/css/app.css +++ b/static/mailing/css/app.css @@ -1,27 +1,64 @@ +/* Datamailer operator UI. Visual foundation follows the DataOps design + system (GitHub Primer neutrals, CMP blue accent, calm row-based content). + Fonts are self-hosted (SIL OFL): Inter variable and IBM Plex Mono. */ +@font-face { + font-family: "Inter"; + font-style: normal; + font-weight: 400 700; + font-display: swap; + src: url("../fonts/inter-var.woff2") format("woff2"); +} + +@font-face { + font-family: "IBM Plex Mono"; + font-style: normal; + font-weight: 400; + font-display: swap; + src: url("../fonts/ibm-plex-mono-400.woff2") format("woff2"); +} + +@font-face { + font-family: "IBM Plex Mono"; + font-style: normal; + font-weight: 500; + font-display: swap; + src: url("../fonts/ibm-plex-mono-500.woff2") format("woff2"); +} + :root { color-scheme: light; - --dm-color-text: #1f2328; - --dm-color-muted: #656d76; + --dm-color-text: #24292f; + --dm-color-heading: #0f172a; + --dm-color-muted: #57606a; + --dm-color-faint: #6e7681; --dm-color-border: #d0d7de; + --dm-color-border-strong: #afb8c1; --dm-color-background: #ffffff; --dm-color-surface: #f6f8fa; - --dm-color-surface-strong: #eaeef2; - --dm-color-focus: #0969da66; - --dm-color-primary: #0969da; - --dm-color-primary-hover: #0757b8; - --dm-color-success: #1a7f37; - --dm-color-success-surface: #dafbe1; - --dm-color-success-border: #aceebb; - --dm-color-warning: #9a6700; + --dm-color-surface-strong: #eef1f4; + --dm-color-accent-soft: #edf5ff; + --dm-color-focus: rgba(49, 95, 143, 0.6); + --dm-color-primary: #315f8f; + --dm-color-primary-hover: #244d78; + --dm-color-link: #315f8f; + --dm-color-link-hover: #244d78; + --dm-color-on-primary: #ffffff; + --dm-color-success: #116329; + --dm-color-success-surface: #e6f6ea; + --dm-color-success-border: #b7dfc2; + --dm-color-warning: #7a5c00; --dm-color-warning-surface: #fff8c5; - --dm-color-warning-border: #eac54f; + --dm-color-warning-border: #d4a72c; --dm-color-danger: #cf222e; --dm-color-danger-hover: #a40e26; - --dm-color-danger-surface: #ffebe9; - --dm-color-danger-border: #ffcecb; + --dm-color-danger-surface: #fff0ee; + --dm-color-danger-border: rgba(207, 34, 46, 0.24); + --dm-color-info: #315f8f; + --dm-color-info-surface: #edf5ff; + --dm-color-info-border: rgba(49, 95, 143, 0.22); --dm-color-neutral: #57606a; --dm-color-neutral-surface: #f6f8fa; - --dm-color-on-primary: #ffffff; + --dm-color-neutral-border: #d0d7de; --dm-space-1: 4px; --dm-space-2: 8px; --dm-space-3: 12px; @@ -30,20 +67,23 @@ --dm-space-6: 24px; --dm-space-8: 32px; --dm-radius-sm: 6px; - --dm-radius-md: 8px; - --dm-font-size-sm: 13px; - --dm-font-size-base: 15px; - --dm-font-size-lg: 20px; - --dm-font-size-xl: 28px; - --dm-control-height: 38px; + --dm-radius-md: 6px; + --dm-font-sans: "Inter", ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; + --dm-font-mono: "IBM Plex Mono", ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; + --dm-font-size-sm: 12px; + --dm-font-size-base: 14px; + --dm-font-size-lg: 16px; + --dm-font-size-xl: 32px; + --dm-control-height: 34px; --dm-content-width: 1120px; + --dm-sidebar-width: 268px; --border: var(--dm-color-border); --ink: var(--dm-color-text); --muted: var(--dm-color-muted); --surface: var(--dm-color-surface); --surface-strong: var(--dm-color-surface-strong); - --accent: var(--dm-color-primary); + --accent: var(--dm-color-link); --danger: var(--dm-color-danger); --success: var(--dm-color-success); } @@ -51,54 +91,74 @@ :root[data-theme="dark"] { color-scheme: dark; --dm-color-text: #e6edf3; + --dm-color-heading: #e6edf3; --dm-color-muted: #8b949e; + --dm-color-faint: #6e7681; --dm-color-border: #30363d; + --dm-color-border-strong: #484f58; --dm-color-background: #0d1117; --dm-color-surface: #161b22; --dm-color-surface-strong: #21262d; - --dm-color-focus: #58a6ff66; - --dm-color-primary: #58a6ff; - --dm-color-primary-hover: #79c0ff; - --dm-color-success: #56d364; - --dm-color-success-surface: #12251a; - --dm-color-success-border: #2ea043; - --dm-color-warning: #e3b341; - --dm-color-warning-surface: #2d2305; - --dm-color-warning-border: #9e6a03; + --dm-color-accent-soft: rgba(77, 127, 168, 0.22); + --dm-color-focus: rgba(139, 183, 223, 0.6); + --dm-color-primary: #4d7fa8; + --dm-color-primary-hover: #6d99c2; + --dm-color-link: #8bb7df; + --dm-color-link-hover: #a7cbed; + --dm-color-on-primary: #ffffff; + --dm-color-success: #7ee787; + --dm-color-success-surface: rgba(31, 111, 59, 0.3); + --dm-color-success-border: rgba(35, 134, 54, 0.4); + --dm-color-warning: #f0d98c; + --dm-color-warning-surface: #2d2516; + --dm-color-warning-border: rgba(187, 128, 9, 0.5); --dm-color-danger: #ff7b72; --dm-color-danger-hover: #ffa198; - --dm-color-danger-surface: #2d1517; - --dm-color-danger-border: #8e2a2f; + --dm-color-danger-surface: rgba(218, 54, 51, 0.2); + --dm-color-danger-border: rgba(218, 54, 51, 0.4); + --dm-color-info: #8bb7df; + --dm-color-info-surface: rgba(77, 127, 168, 0.22); + --dm-color-info-border: rgba(77, 127, 168, 0.42); --dm-color-neutral: #8b949e; --dm-color-neutral-surface: #21262d; - --dm-color-on-primary: #0d1117; + --dm-color-neutral-border: #30363d; } @media (prefers-color-scheme: dark) { :root:not([data-theme]) { color-scheme: dark; --dm-color-text: #e6edf3; + --dm-color-heading: #e6edf3; --dm-color-muted: #8b949e; + --dm-color-faint: #6e7681; --dm-color-border: #30363d; + --dm-color-border-strong: #484f58; --dm-color-background: #0d1117; --dm-color-surface: #161b22; --dm-color-surface-strong: #21262d; - --dm-color-focus: #58a6ff66; - --dm-color-primary: #58a6ff; - --dm-color-primary-hover: #79c0ff; - --dm-color-success: #56d364; - --dm-color-success-surface: #12251a; - --dm-color-success-border: #2ea043; - --dm-color-warning: #e3b341; - --dm-color-warning-surface: #2d2305; - --dm-color-warning-border: #9e6a03; + --dm-color-accent-soft: rgba(77, 127, 168, 0.22); + --dm-color-focus: rgba(139, 183, 223, 0.6); + --dm-color-primary: #4d7fa8; + --dm-color-primary-hover: #6d99c2; + --dm-color-link: #8bb7df; + --dm-color-link-hover: #a7cbed; + --dm-color-on-primary: #ffffff; + --dm-color-success: #7ee787; + --dm-color-success-surface: rgba(31, 111, 59, 0.3); + --dm-color-success-border: rgba(35, 134, 54, 0.4); + --dm-color-warning: #f0d98c; + --dm-color-warning-surface: #2d2516; + --dm-color-warning-border: rgba(187, 128, 9, 0.5); --dm-color-danger: #ff7b72; --dm-color-danger-hover: #ffa198; - --dm-color-danger-surface: #2d1517; - --dm-color-danger-border: #8e2a2f; + --dm-color-danger-surface: rgba(218, 54, 51, 0.2); + --dm-color-danger-border: rgba(218, 54, 51, 0.4); + --dm-color-info: #8bb7df; + --dm-color-info-surface: rgba(77, 127, 168, 0.22); + --dm-color-info-border: rgba(77, 127, 168, 0.42); --dm-color-neutral: #8b949e; --dm-color-neutral-surface: #21262d; - --dm-color-on-primary: #0d1117; + --dm-color-neutral-border: #30363d; } } @@ -119,8 +179,12 @@ body { body { margin: 0; color: var(--dm-color-text); - font: var(--dm-font-size-base) / 1.5 system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; + font-family: var(--dm-font-sans); + font-size: var(--dm-font-size-base); + line-height: 1.5; background: var(--dm-color-background); + -webkit-font-smoothing: antialiased; + text-rendering: optimizeLegibility; } header, @@ -136,18 +200,21 @@ nav, gap: var(--dm-space-5); min-height: 52px; margin: 0 auto; - padding: 0 var(--dm-space-5); + padding: 0 var(--dm-space-4); } nav a, .app-nav a { - color: var(--dm-color-text); + color: var(--dm-color-muted); + font-size: 14px; + font-weight: 500; text-decoration: none; } nav a:hover, .app-nav a:hover { - text-decoration: underline; + color: var(--dm-color-text); + text-decoration: none; } nav .brand, @@ -155,7 +222,10 @@ nav .brand, display: inline-flex; align-items: center; min-height: 52px; + color: var(--dm-color-heading); + font-size: 15px; font-weight: 700; + letter-spacing: -0.01em; } .top-nav-actions { @@ -167,28 +237,29 @@ nav .brand, .top-nav-actions a { color: var(--dm-color-muted); - font-size: var(--dm-font-size-sm); + font-size: 13px; } .theme-toggle { width: auto; - min-height: 32px; - padding: 4px 9px; + min-height: 30px; + padding: 3px 10px; color: var(--dm-color-muted); border-color: var(--dm-color-border); background: var(--dm-color-background); font-size: var(--dm-font-size-sm); + font-weight: 500; } .theme-toggle:hover { color: var(--dm-color-text); - border-color: var(--dm-color-muted); + border-color: var(--dm-color-border-strong); background: var(--dm-color-surface); } .app-nav a[aria-current="page"] { - color: var(--dm-color-primary); - font-weight: 700; + color: var(--dm-color-link); + font-weight: 600; } .client-switcher { @@ -198,30 +269,33 @@ nav .brand, .client-switcher label { color: var(--dm-color-muted); - font-size: var(--dm-font-size-sm); + font-size: 11px; + font-weight: 600; + letter-spacing: 0.07em; + text-transform: uppercase; } .client-switcher select { width: 100%; - height: 36px; + height: 34px; padding: 0 var(--dm-space-2); } main, .app-main { max-width: var(--dm-content-width); - padding: var(--dm-space-8) var(--dm-space-5); + padding: var(--dm-space-8) var(--dm-space-6) 64px; } .app-shell { display: grid; - grid-template-columns: 196px minmax(0, 1fr); + grid-template-columns: var(--dm-sidebar-width) minmax(0, 1fr); min-height: calc(100vh - 53px); - background: var(--dm-color-surface); + background: var(--dm-color-background); } .app-shell.sidebar-collapsed { - grid-template-columns: 40px minmax(0, 1fr); + grid-template-columns: 44px minmax(0, 1fr); } .app-sidebar { @@ -230,24 +304,29 @@ main, height: calc(100vh - 53px); overflow: auto; border-right: 1px solid var(--dm-color-border); - background: var(--dm-color-background); + background: var(--dm-color-surface); } .sidebar-toggle { - min-height: 24px; - height: 24px; - padding: 0; - border: 0; + display: inline-flex; + align-items: center; + justify-content: center; + min-height: 28px; + padding: 2px 8px; + border: 1px solid transparent; + border-radius: var(--dm-radius-sm); background: transparent; color: var(--dm-color-muted); - font-size: 12px; + font-size: var(--dm-font-size-sm); + font-weight: 500; cursor: pointer; } .sidebar-toggle:hover { color: var(--dm-color-text); - background: transparent; - text-decoration: underline; + border-color: var(--dm-color-border); + background: var(--dm-color-background); + text-decoration: none; } .sidebar-toggle-closed { @@ -255,18 +334,20 @@ main, } .sidebar-inner { - padding: var(--dm-space-1) var(--dm-space-2) var(--dm-space-3); + padding: var(--dm-space-3) var(--dm-space-3) var(--dm-space-4); } .sidebar-titlebar { display: flex; justify-content: flex-end; - margin-bottom: var(--dm-space-1); + margin-bottom: var(--dm-space-2); } .sidebar-context { - padding-bottom: var(--dm-space-2); - border-bottom: 1px solid var(--dm-color-border); + padding: var(--dm-space-3); + border: 1px solid var(--dm-color-border); + border-radius: var(--dm-radius-md); + background: var(--dm-color-background); } .sidebar-context-header { @@ -301,48 +382,50 @@ main, .sidebar-nav { display: grid; - gap: var(--dm-space-3); - margin-top: var(--dm-space-2); + gap: var(--dm-space-4); + margin-top: var(--dm-space-4); } .sidebar-nav-group { display: grid; - gap: var(--dm-space-1); + gap: 2px; } .sidebar-nav-label { - padding: 0 var(--dm-space-1) var(--dm-space-1); + margin: 0 8px var(--dm-space-1); color: var(--dm-color-muted); font-size: 11px; - font-weight: 700; + font-weight: 600; + letter-spacing: 0.07em; text-transform: uppercase; } .sidebar-nav a { display: block; - min-height: 0; - padding: 7px var(--dm-space-2); - border: 1px solid transparent; + min-height: 34px; + padding: 6px 10px; + border: 0; border-radius: var(--dm-radius-sm); color: var(--dm-color-text); + font-size: 14px; + font-weight: 500; text-decoration: none; } .sidebar-nav a:hover { - border-color: var(--dm-color-border); - background: var(--dm-color-surface); + background: var(--dm-color-surface-strong); text-decoration: none; } .sidebar-nav a[aria-current="page"] { - border-color: var(--dm-color-border); - background: var(--dm-color-surface); - box-shadow: inset 3px 0 0 var(--dm-color-primary); + background: var(--dm-color-accent-soft); + color: var(--dm-color-link); + font-weight: 600; } .sidebar-collapsed .sidebar-inner { display: block; - padding: var(--dm-space-1); + padding: var(--dm-space-2); } .sidebar-collapsed .sidebar-titlebar { @@ -364,7 +447,7 @@ main, } .sidebar-collapsed .sidebar-toggle { - width: 28px; + width: 30px; padding: 0; writing-mode: vertical-rl; } @@ -372,27 +455,34 @@ main, .app-main { width: 100%; max-width: var(--dm-content-width); - margin: var(--dm-space-5) auto; - border: 1px solid var(--dm-color-border); - border-radius: var(--dm-radius-md); - background: var(--dm-color-background); + margin: 0 auto; + border: 0; + border-radius: 0; + background: transparent; } h1 { - margin: 0 0 var(--dm-space-3); + margin: 0 0 var(--dm-space-2); + color: var(--dm-color-heading); font-size: var(--dm-font-size-xl); + font-weight: 600; line-height: 1.2; + letter-spacing: -0.02em; } h2 { - margin: 28px 0 var(--dm-space-3); + margin: var(--dm-space-6) 0 var(--dm-space-2); + color: var(--dm-color-heading); font-size: var(--dm-font-size-lg); + font-weight: 600; line-height: 1.25; } h3 { - margin: 18px 0 var(--dm-space-2); - font-size: 16px; + margin: var(--dm-space-4) 0 var(--dm-space-2); + color: var(--dm-color-heading); + font-size: 14px; + font-weight: 600; } p { @@ -400,7 +490,11 @@ p { } a { - color: var(--dm-color-primary); + color: var(--dm-color-link); +} + +a:hover { + color: var(--dm-color-link-hover); } a:focus-visible, @@ -416,30 +510,40 @@ textarea:focus-visible, .page-header, .section-header { display: flex; + flex-wrap: wrap; justify-content: space-between; align-items: flex-start; - gap: 18px; - margin-bottom: 22px; + gap: var(--dm-space-3); + margin-bottom: var(--dm-space-6); +} + +.page-header p { + max-width: 60ch; + margin: 0; } .section-header { - margin-top: 28px; + margin-top: var(--dm-space-6); margin-bottom: var(--dm-space-3); } +.section-header p { + margin: 0; +} + .breadcrumbs { display: flex; flex-wrap: wrap; gap: var(--dm-space-2); margin: 0 0 var(--dm-space-4); color: var(--dm-color-muted); - font-size: var(--dm-font-size-sm); + font-size: 13px; } .section, .detail-section { min-width: 0; - margin: 28px 0; + margin: var(--dm-space-6) 0; } .muted, @@ -448,13 +552,13 @@ textarea:focus-visible, } .helptext { - margin-top: var(--dm-space-1); + margin-top: 2px; font-size: var(--dm-font-size-sm); } .readonly-field { min-height: var(--dm-control-height); - padding: 8px 10px; + padding: 6px 10px; border: 1px solid var(--dm-color-border); border-radius: var(--dm-radius-sm); background: var(--dm-color-surface); @@ -467,15 +571,21 @@ textarea:focus-visible, display: grid; grid-template-columns: repeat(auto-fit, minmax(150px, 1fr)); gap: var(--dm-space-3); - margin: 18px 0 var(--dm-space-6); + margin: var(--dm-space-4) 0 var(--dm-space-6); } .detail-grid { grid-template-columns: repeat(auto-fit, minmax(220px, 1fr)); } -.stat, +/* Summary counts read as data: quiet bordered cells with mono values. + Cells keep their own hairline borders so wrapped rows stay clean. */ +.stats-grid { + gap: var(--dm-space-3); +} + .meta-item, +.stat, .empty-state { border: 1px solid var(--dm-color-border); border-radius: var(--dm-radius-md); @@ -485,13 +595,14 @@ textarea:focus-visible, .stat-value { display: block; - font-size: var(--dm-space-6); - font-weight: 700; - line-height: 1.15; + font-family: var(--dm-font-mono); + font-size: 22px; + font-weight: 500; + line-height: 1.2; } a.stat-link { - color: var(--dm-color-primary); + color: var(--dm-color-link); text-decoration: none; } @@ -503,10 +614,15 @@ a.stat-link:focus { .stat-label, .meta-label { display: block; + margin-top: 2px; color: var(--dm-color-muted); font-size: var(--dm-font-size-sm); } +.stat .helptext { + margin-top: 0; +} + .meta-item dd { min-width: 0; margin: 0; @@ -516,33 +632,39 @@ a.stat-link:focus { .badge { display: inline-flex; align-items: center; - min-height: var(--dm-space-6); + min-height: 20px; padding: 0 var(--dm-space-2); + border: 1px solid var(--dm-color-neutral-border); border-radius: 999px; color: var(--dm-color-neutral); - background: var(--dm-color-surface-strong); + background: var(--dm-color-neutral-surface); + font-family: var(--dm-font-sans); font-size: var(--dm-font-size-sm); - font-weight: 600; + font-weight: 500; white-space: nowrap; } .badge.success { color: var(--dm-color-success); + border-color: var(--dm-color-success-border); background: var(--dm-color-success-surface); } .badge.warning { color: var(--dm-color-warning); + border-color: var(--dm-color-warning-border); background: var(--dm-color-warning-surface); } .badge.danger { color: var(--dm-color-danger); + border-color: var(--dm-color-danger-border); background: var(--dm-color-danger-surface); } .badge.neutral { color: var(--dm-color-neutral); + border-color: var(--dm-color-neutral-border); background: var(--dm-color-neutral-surface); } @@ -601,16 +723,41 @@ a.stat-link:focus { .metric-value { display: block; - color: var(--dm-color-text); - font-weight: 700; + font-family: var(--dm-font-mono); + color: var(--dm-color-heading); + font-weight: 500; } +/* Attention queues render as one bordered list with hairline-divided rows. */ .compact-list, .form-stack { display: grid; gap: var(--dm-space-3); } +.compact-list { + gap: 0; + border: 1px solid var(--dm-color-border); + border-radius: var(--dm-radius-md); + background: var(--dm-color-background); + overflow: hidden; +} + +.compact-list .compact-row { + border: 0; + border-bottom: 1px solid var(--dm-color-border); + border-radius: 0; +} + +.compact-list .compact-row:last-child { + border-bottom: 0; +} + +.compact-list .compact-row:hover { + background: var(--dm-color-surface); + border-color: var(--dm-color-border); +} + .manage-form-grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(260px, 1fr)); @@ -623,7 +770,8 @@ a.stat-link:focus { .compact-row { display: grid; - gap: var(--dm-space-2); + gap: var(--dm-space-1); + transition: background 120ms ease; } a.compact-row { @@ -632,8 +780,9 @@ a.compact-row { } a.compact-row:hover { - border-color: var(--dm-color-muted); + border-color: var(--dm-color-border-strong); background: var(--dm-color-surface); + text-decoration: none; } .compact-row-main, @@ -645,6 +794,11 @@ a.compact-row:hover { gap: var(--dm-space-2); } +.compact-row-meta { + color: var(--dm-color-muted); + font-size: 13px; +} + .compact-row-main strong, .activity-timeline strong { overflow-wrap: anywhere; @@ -658,14 +812,14 @@ a.compact-row:hover { .secondary-section summary { color: var(--dm-color-text); cursor: pointer; - font-weight: 700; + font-weight: 600; } .inline-details summary { - color: var(--dm-color-primary); + color: var(--dm-color-link); cursor: pointer; - font-size: var(--dm-font-size-sm); - font-weight: 700; + font-size: 13px; + font-weight: 600; } .inline-details[open] summary { @@ -681,7 +835,7 @@ a.compact-row:hover { display: flex; flex-wrap: wrap; align-items: end; - gap: 10px; + gap: var(--dm-space-2); margin: var(--dm-space-4) 0; } @@ -704,6 +858,7 @@ a.compact-row:hover { align-items: center; gap: 6px; color: var(--dm-color-text); + font-size: 13px; font-weight: 400; } @@ -756,7 +911,12 @@ a.compact-row:hover { padding: var(--dm-space-3); color: var(--dm-color-text); cursor: pointer; - font-weight: 700; + font-weight: 600; +} + +.advanced-panel summary:hover, +.secondary-section summary:hover { + color: var(--dm-color-heading); } .advanced-panel[open] { @@ -793,7 +953,7 @@ a.compact-row:hover { display: grid; gap: var(--dm-space-1); color: var(--dm-color-muted); - font-size: var(--dm-font-size-sm); + font-size: 13px; } .actions, @@ -801,7 +961,7 @@ a.compact-row:hover { display: flex; flex-wrap: wrap; justify-content: flex-end; - gap: 10px; + gap: var(--dm-space-2); } .actions form, @@ -812,7 +972,7 @@ a.compact-row:hover { .messages { display: grid; gap: var(--dm-space-2); - margin: 0 0 18px; + margin: 0 0 var(--dm-space-4); padding: 0; list-style: none; } @@ -821,8 +981,14 @@ a.compact-row:hover { .alert { border: 1px solid var(--dm-color-border); border-radius: var(--dm-radius-md); - padding: 10px var(--dm-space-3); + padding: var(--dm-space-2) var(--dm-space-3); background: var(--dm-color-surface); + color: var(--dm-color-text); +} + +.message strong, +.alert strong { + color: inherit; } .message.success, @@ -887,6 +1053,7 @@ a.compact-row:hover { gap: var(--dm-space-2); color: var(--dm-color-text); font-size: var(--dm-font-size-base); + font-weight: 400; } .checkbox-field input { @@ -899,6 +1066,7 @@ a.compact-row:hover { .errorlist { margin: 6px 0 0; color: var(--dm-color-danger); + font-size: 13px; } .errorlist { @@ -907,8 +1075,9 @@ a.compact-row:hover { label { display: block; - color: var(--dm-color-muted); - font-size: var(--dm-font-size-sm); + margin-bottom: var(--dm-space-1); + color: var(--dm-color-text); + font-size: 13px; font-weight: 600; } @@ -920,9 +1089,30 @@ button, min-height: var(--dm-control-height); border: 1px solid var(--dm-color-border); border-radius: var(--dm-radius-sm); - padding: 7px 10px; - font: inherit; + padding: 5px 10px; + font-family: var(--dm-font-sans); + font-size: var(--dm-font-size-base); background: var(--dm-color-background); + color: var(--dm-color-text); +} + +input::placeholder, +textarea::placeholder { + color: var(--dm-color-faint); +} + +input[type="checkbox"], +input[type="radio"] { + width: auto; + min-height: 0; +} + +input:focus, +select:focus, +textarea:focus { + border-color: var(--dm-color-primary); + outline: none; + box-shadow: 0 0 0 3px var(--dm-color-accent-soft); } button, @@ -930,32 +1120,43 @@ button, display: inline-flex; align-items: center; justify-content: center; + min-height: var(--dm-control-height); + padding: 5px 12px; color: var(--dm-color-on-primary); border-color: var(--dm-color-primary); background: var(--dm-color-primary); + font-size: var(--dm-font-size-base); + font-weight: 600; cursor: pointer; text-decoration: none; } button:hover, .button:hover { + color: var(--dm-color-on-primary); border-color: var(--dm-color-primary-hover); background: var(--dm-color-primary-hover); text-decoration: none; } +button:disabled, +.button[aria-disabled="true"] { + cursor: not-allowed; + opacity: 0.65; +} + .button.secondary, button.secondary { color: var(--dm-color-text); border-color: var(--dm-color-border); - background: var(--dm-color-background); + background: var(--dm-color-surface); } .button.secondary:hover, button.secondary:hover { color: var(--dm-color-text); - border-color: var(--dm-color-muted); - background: var(--dm-color-surface); + border-color: var(--dm-color-border-strong); + background: var(--dm-color-surface-strong); } button.danger, @@ -977,13 +1178,14 @@ button.danger:hover, -webkit-overflow-scrolling: touch; border: 1px solid var(--dm-color-border); border-radius: var(--dm-radius-md); + background: var(--dm-color-background); } table { width: 100%; min-width: 760px; border-collapse: collapse; - background: var(--dm-color-background); + background: transparent; } th, @@ -998,17 +1200,41 @@ th { color: var(--dm-color-muted); background: var(--dm-color-surface); font-size: var(--dm-font-size-sm); - font-weight: 700; + font-weight: 600; + white-space: nowrap; +} + +td { + font-size: 13px; +} + +td strong { + color: var(--dm-color-heading); + font-size: var(--dm-font-size-base); } tr:last-child td { border-bottom: 0; } +.table-wrap tbody tr:hover td { + background: var(--dm-color-surface); +} + .nowrap { white-space: nowrap; } +.table-wrap td.nowrap { + font-family: var(--dm-font-mono); + font-size: 12.5px; +} + +.table-wrap td.nowrap .helptext, +.table-wrap td.nowrap a { + font-family: var(--dm-font-sans); +} + .truncate { max-width: 280px; overflow-wrap: anywhere; @@ -1211,25 +1437,33 @@ tr:last-child td { display: flex; flex-wrap: wrap; align-items: center; - gap: 10px; + gap: var(--dm-space-2); margin: var(--dm-space-4) 0; + color: var(--dm-color-muted); + font-size: 13px; } .timeline { display: grid; - gap: 10px; + gap: var(--dm-space-2); } .timeline-item, .audit-row { - border-left: 4px solid var(--dm-color-primary); - padding: 10px var(--dm-space-3); - background: var(--dm-color-surface); + border: 1px solid var(--dm-color-border); + border-left: 3px solid var(--dm-color-border-strong); + border-radius: var(--dm-radius-sm); + padding: var(--dm-space-2) var(--dm-space-3); + background: var(--dm-color-background); } pre, code { - font-family: ui-monospace, SFMono-Regular, Consolas, "Liberation Mono", monospace; + font-family: var(--dm-font-mono); +} + +code { + font-size: 0.9em; } pre { @@ -1238,13 +1472,11 @@ pre { border-radius: var(--dm-radius-md); padding: var(--dm-space-3); color: var(--dm-color-text); + font-size: 12.5px; + line-height: 1.55; background: var(--dm-color-surface); } -code { - font-size: 0.92em; -} - .api-docs-header { padding-bottom: var(--dm-space-5); border-bottom: 1px solid var(--dm-color-border); @@ -1281,12 +1513,13 @@ code { .api-docs-nav a { color: var(--dm-color-muted); - font-size: var(--dm-font-size-sm); + font-size: 13px; + font-weight: 400; text-decoration: none; } .api-docs-nav a:hover { - color: var(--dm-color-primary); + color: var(--dm-color-link); text-decoration: underline; } @@ -1325,7 +1558,7 @@ code { .api-example-meta dt { color: var(--dm-color-muted); font-size: var(--dm-font-size-sm); - font-weight: 700; + font-weight: 600; } .api-key-meta dd, @@ -1388,9 +1621,9 @@ code { border-bottom: 0; border-radius: var(--dm-radius-sm) var(--dm-radius-sm) 0 0; color: var(--dm-color-muted); - background: var(--dm-color-background); + background: var(--dm-color-surface); font-size: var(--dm-font-size-sm); - font-weight: 700; + font-weight: 600; } .code-block pre { @@ -1485,13 +1718,17 @@ code { min-height: 38px; } + .table-wrap { + scrollbar-width: thin; + } + .client-table, .campaign-table, .audience-table, .audience-member-table { max-width: 100%; scrollbar-width: thin; - box-shadow: inset -14px 0 14px -16px var(--dm-color-muted); + box-shadow: inset -14px 0 14px -16px var(--dm-color-border-strong); } .app-main { @@ -1499,6 +1736,7 @@ code { margin: 0; border: 0; border-radius: 0; + padding: var(--dm-space-5) var(--dm-space-4) var(--dm-space-8); } .api-docs-layout, @@ -1514,7 +1752,7 @@ code { @media (max-width: 640px) { main, .app-main { - padding: var(--dm-space-6) 14px; + padding: var(--dm-space-5) 14px; } nav, @@ -1524,6 +1762,10 @@ code { padding-left: 14px; } + h1 { + font-size: 22px; + } + .page-header, .section-header { display: grid; @@ -1575,6 +1817,11 @@ code { width: 100%; } + input[type="checkbox"], + input[type="radio"] { + width: auto; + } + .top-nav-actions .theme-toggle { width: auto; } @@ -1605,7 +1852,9 @@ code { margin: 0 0 var(--dm-space-4); color: var(--dm-color-muted); font-size: var(--dm-font-size-sm); - font-weight: 700; + font-weight: 600; + letter-spacing: 0.07em; + text-transform: uppercase; } .unsubscribe-state { @@ -1644,7 +1893,7 @@ code { .unsubscribe-context dt { color: var(--dm-color-muted); font-size: var(--dm-font-size-sm); - font-weight: 700; + font-weight: 600; } .unsubscribe-context dd { @@ -1668,7 +1917,7 @@ code { .unsubscribe-options legend { margin-bottom: var(--dm-space-3); color: var(--dm-color-text); - font-weight: 700; + font-weight: 600; } .unsubscribe-option { @@ -1684,9 +1933,13 @@ code { cursor: pointer; } +.unsubscribe-option:hover { + border-color: var(--dm-color-border-strong); +} + .unsubscribe-option:focus-within { border-color: var(--dm-color-primary); - box-shadow: 0 0 0 3px var(--dm-color-focus); + box-shadow: 0 0 0 3px var(--dm-color-accent-soft); } .unsubscribe-option input { diff --git a/static/mailing/fonts/LICENSE.md b/static/mailing/fonts/LICENSE.md new file mode 100644 index 0000000..05e9cc1 --- /dev/null +++ b/static/mailing/fonts/LICENSE.md @@ -0,0 +1,12 @@ +# Font licenses + +Both font families in this directory are self-hosted under the SIL Open Font +License and must not trigger third-party requests. + +- `inter-var.woff2` — Inter (variable, weights 400-700), Copyright The Inter + Project Authors, SIL Open Font License 1.1. +- `ibm-plex-mono-400.woff2`, `ibm-plex-mono-500.woff2` — IBM Plex Mono, + Copyright IBM Corp., SIL Open Font License 1.1. + +Full license texts: https://github.com/rsms/inter/blob/master/LICENSE.txt and +https://github.com/IBM/plex/blob/master/LICENSE.txt. diff --git a/static/mailing/fonts/ibm-plex-mono-400.woff2 b/static/mailing/fonts/ibm-plex-mono-400.woff2 new file mode 100644 index 0000000..0804aaf Binary files /dev/null and b/static/mailing/fonts/ibm-plex-mono-400.woff2 differ diff --git a/static/mailing/fonts/ibm-plex-mono-500.woff2 b/static/mailing/fonts/ibm-plex-mono-500.woff2 new file mode 100644 index 0000000..090f82f Binary files /dev/null and b/static/mailing/fonts/ibm-plex-mono-500.woff2 differ diff --git a/static/mailing/fonts/inter-var.woff2 b/static/mailing/fonts/inter-var.woff2 new file mode 100644 index 0000000..d15208d Binary files /dev/null and b/static/mailing/fonts/inter-var.woff2 differ diff --git a/templates/base.html b/templates/base.html index cb9bf13..6191877 100644 --- a/templates/base.html +++ b/templates/base.html @@ -76,6 +76,9 @@ Contacts + + Inbound mail +
Configure @@ -85,6 +88,9 @@ Templates + + Receiving addresses + {% if operator_active_client %} Settings diff --git a/templates/mailing/operator/inbound_address_form.html b/templates/mailing/operator/inbound_address_form.html new file mode 100644 index 0000000..f6ca576 --- /dev/null +++ b/templates/mailing/operator/inbound_address_form.html @@ -0,0 +1,65 @@ +{% extends "base.html" %} + +{% block title %}New receiving address | Relay{% endblock %} + +{% block content %} +
+
+

New receiving address

+

+ The address starts receiving on the next message delivered to it. No + Terraform change and no redeploy is involved. +

+
+
+ Cancel +
+
+ +
+
+ {% csrf_token %} +
+ + +

The part before the @. Lowercased on save.

+
+
+ + +

+ Must already receive at this root. Creating an address here does not + change DNS or SES, so a domain nobody points at Relay will simply + never arrive. +

+
+
+ + +

Only you see this. Useful when several addresses exist for the same purpose.

+
+
+ +
+
+
+ + {% if domain_hint %} +
+
+
+

Currently receiving through the environment

+

+ These routes work today and keep working. An address created here takes + precedence over them for the same address. +

+
+
+
    + {% for address in domain_hint %} +
  • {{ address }}
    • + {% endfor %} +
+
+ {% endif %} +{% endblock %} diff --git a/templates/mailing/operator/inbound_address_list.html b/templates/mailing/operator/inbound_address_list.html new file mode 100644 index 0000000..de4a468 --- /dev/null +++ b/templates/mailing/operator/inbound_address_list.html @@ -0,0 +1,148 @@ +{% extends "base.html" %} + +{% block title %}Receiving addresses | Relay{% endblock %} + +{% block content %} +
+
+

Receiving addresses

+

+ Addresses Relay receives for. Creating one here is the whole change: no + Terraform edit, no apply, no redeploy. +

+
+
+ Inbound mail + New address +
+
+ +
+
+
Active
+
{{ active_count }}
+
+
+
Retired
+
{{ retired_count }}
+
+
+ +
+
+
+

Addresses

+

+ A retired address stops receiving but keeps its mail. Messages are + discarded, not rerouted, when they arrive for an address that does not + exist. +

+
+
+ {% if addresses %} +
+ + + + + + + + + + + + {% for row in addresses %} + + + + + + + + {% endfor %} + +
AddressNoteMessagesStateActions
{{ row.address }}{{ row.note|default:"-" }} + {% if row.message_count %} + {{ row.message_count }} + {% else %} + 0 + {% endif %} + + {% if row.is_active %} + Receiving + {% else %} + Retired + {% endif %} + +
+ {% csrf_token %} + +
+
+
+ {% else %} +
+ No addresses yet. +

+ Until an address exists here, inbound mail falls back to + INBOUND_EMAIL_ROUTES from the environment. + {% if routes %} + Currently configured there: {{ routes|join:", " }}. + {% endif %} +

+
+ {% endif %} +
+ +
+
+
+

Blocked senders

+

+ Checked before storage. A blocked message is recorded without its body + so the block is auditable, and so unblocking does not resurrect + something nobody read. +

+
+
+ {% if blocked_rules %} +
+ + + + + + + + + + + + {% for rule in blocked_rules %} + + + + + + + + {% endfor %} + +
ScopeValueReasonDiscardedActions
{{ rule.get_scope_display }}{{ rule.value }}{{ rule.reason|default:"-" }}{{ rule.blocked_message_count }} +
+ {% csrf_token %} + +
+
+
+ {% else %} +
+ Nothing is blocked. +

Mark a message as spam from the mailbox to add a rule here.

+
+ {% endif %} +
+{% endblock %} diff --git a/templates/mailing/operator/inbound_list.html b/templates/mailing/operator/inbound_list.html new file mode 100644 index 0000000..cb4c9c3 --- /dev/null +++ b/templates/mailing/operator/inbound_list.html @@ -0,0 +1,173 @@ +{% extends "base.html" %} + +{% block title %}Inbound | Relay{% endblock %} + +{% block content %} +
+
+

Inbound mail

+

Mail received for Relay-owned addresses. Not tied to a client: a receiving address belongs to the service.

+
+
+ Addresses +
+
+ +
+
+
Unread
+
{{ counts.received|default:0 }}
+
+
+
Read
+
{{ counts.read|default:0 }}
+
+
+
Blocked
+
{{ counts.blocked|default:0 }}
+
+
+
Senders waiting
+
{{ sender_count|default:0 }}
+
+
+ +
+
+
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+
+
+ + {% if rows %} +
+
+
+

Messages

+

{{ total }} message{{ total|pluralize }}.

+
+
+
+ + + + + + + + + + + + {% for row in rows %} + + + + + + + + {% endfor %} + +
FromSubjectToReceivedSignal
+ {{ row.sender_address|default:row.from_header|default:"Unknown sender" }} + {% if row.sender_domain %} +
{{ row.sender_domain }}
+ {% endif %} + 		{{ row.subject|default:"(no subject)" }}{{ row.recipient_address|default:"-" }}{{ row.created_at|date:"Y-m-d H:i" }} + {% if row.is_spam %} + SES: spam + {% elif row.spam_verdict %} + SES: {{ row.spam_verdict }} + {% elif row.attachment_count %} + {{ row.attachment_count }} attachment{{ row.attachment_count|pluralize }} + {% else %} + - + {% endif %} +
+
+ {% if pages > 1 %} + + {% endif %} +
+ {% else %} +
+ {% if narrowed %} + No messages match this filter. +

Nothing here is a statement about whether mail arrived; only about this filter.

+ {% else %} + No unread mail. +

Mail received for a Relay address will appear here. Until a receiving domain points at Relay, nothing arrives.

+ {% endif %} +
+ {% endif %} + + {% if blocked_senders %} +
+
+
+

Blocked senders

+

Discarded at ingest, so they never reach the mailbox. Managed on the addresses page.

+
+
+
+ + + + + + + + + + + {% for rule in blocked_senders %} + + + + + + + {% endfor %} + +
ScopeValueReasonBlocked
{{ rule.get_scope_display }}{{ rule.value }}{{ rule.reason|default:"-" }}{{ rule.created_at|date:"Y-m-d" }}
+
+
+ {% endif %} +{% endblock %} diff --git a/templates/mailing/operator/inbound_message_detail.html b/templates/mailing/operator/inbound_message_detail.html new file mode 100644 index 0000000..2903552 --- /dev/null +++ b/templates/mailing/operator/inbound_message_detail.html @@ -0,0 +1,139 @@ +{% extends "base.html" %} + +{% block title %}{{ message.subject|default:"Inbound message" }} | Relay{% endblock %} + +{% block content %} + + +
+
+

{{ message.subject|default:"(no subject)" }}

+

+ From {{ message.sender_address|default:message.from_header|default:"unknown sender" }} + to {{ message.recipient_address|default:"an unknown address" }} + · received {{ message.created_at|date:"Y-m-d H:i" }} +

+
+
+ {{ badge.label }} +
+
+ +
+
+
+

Summary

+
+
+
From
{{ message.from_header|default:"-" }}
+
Sender address
{{ message.sender_address|default:"-" }}
+
To
{{ message.to_header|default:"-" }}
+
Cc
{{ message.cc_header|default:"-" }}
+
Claimed sent
{{ message.sent_at|date:"Y-m-d H:i"|default:"-" }}
+
Size
{{ message.size_bytes }} bytes
+
Attachments
{{ message.attachment_count }}
+
Message-ID
{{ message.message_id|default:"-" }}
+
+
+ +
+
+

Authentication and spam signals

+
+
+
SES spam
{{ message.spam_verdict|default:"-" }}
+
SES virus
{{ message.virus_verdict|default:"-" }}
+
SPF
{{ message.spf_verdict|default:"-" }}
+
DKIM
{{ message.dkim_verdict|default:"-" }}
+
DMARC
{{ message.dmarc_verdict|default:"-" }}
+
+

+ A verdict is the sending domain's own claim, not a judgement. SES reporting + spam means SES thinks so; it does not mean the message is. +

+
+
+ +
+
+

Body

+
+ {% if message.state == "blocked" %} +
+ This sender is blocked, so the body was not kept. That is the point of + blocking: future mail is discarded rather than filed. +
+ {% elif body %} + 
{{ body }}
+ {% elif message.has_body %} +
+ The body is stored in object storage and could not be read right now. +
+ {% else %} +
+ No text body. +

This message had no text/plain part. The raw MIME is retained in object storage.

+
+ {% endif %} +
+ +
+
+

Manage this sender

+
+

+ Blocking stops future mail from this sender at ingest. It does not delete + what is already stored, and it does not send anything back to the sender. +

+
+ {% if message.state == "blocked" %} +
+ {% csrf_token %} + + +
+ {% elif message.sender_address %} +
+ {% csrf_token %} + + + +
+ {% if message.sender_domain %} +
+ {% csrf_token %} + + + +
+ {% endif %} + {% else %} +

This message has no parsable sender address, so there is nothing to block.

+ {% endif %} +
+ {% if blocked_rule %} +
+ Blocked by the rule {{ blocked_rule.value }} + ({{ blocked_rule.get_scope_display|lower }}), created {{ blocked_rule.created_at|date:"Y-m-d H:i" }}. +
+ {% endif %} +
+ +
+ Storage locations +
+
Raw MIME bucket
{{ message.raw_bucket|default:"-" }}
+
Raw MIME key
{{ message.raw_key|default:"-" }}
+
Text body key
{{ message.body_text_key|default:"-" }}
+
HTML body key
{{ message.body_html_key|default:"-" }}
+
+

+ The raw message and both body parts live in object storage, not the + database. Headers are indexed because the console lists and searches on + them; bodies are read one at a time, by an operator who has decided to + open one. +

+
+{% endblock %} diff --git a/tests/test_dev_auto_login.py b/tests/test_dev_auto_login.py new file mode 100644 index 0000000..a24e55f --- /dev/null +++ b/tests/test_dev_auto_login.py @@ -0,0 +1,24 @@ +import pytest +from django.contrib.auth import get_user_model + +pytestmark = pytest.mark.django_db + + +@pytest.fixture +def admin_user(): + return get_user_model().objects.create_superuser("admin", "admin@example.com", "admin") + + +def test_debug_pre_authenticates_requests_as_superuser(client, settings, admin_user): + settings.DEBUG = True + response = client.get("/") + assert response.status_code == 200 + assert response.context["user"] == admin_user + + +def test_without_debug_the_operator_ui_still_requires_login(client, admin_user): + # The test suite runs with DEBUG off, like a deployed environment: the + # middleware must stay dormant there and the staff gate must hold. + response = client.get("/") + assert response.status_code == 302 + assert response["Location"] == "/admin/login/?next=/"