Files
CCMA/tests/test_sepa.py
T
Marcel Peterkau 5fc6d7aec9 Stop rolling back reminders/SEPA mails after a successful send
The previous fix reverted the reminder to "draft" on any failure after
mark_reminder_sent, including failures that happened after the mail had
already been handed to send_via_smtp or appended to an IMAP folder. That
made a successful send followed by a Sent-folder-copy or archive-move
failure look like nothing was sent, inviting a duplicate send/fee booking
on retry -- the same "sent" flag needs to be preserved once delivery is
no longer reversible, per follow-up review.

reminder_mail.generate_and_send_reminder_mail and
sepa_mail.generate_debit_mails now track whether the mail actually left
the building (SMTP accepted / IMAP append succeeded / local file written)
separately from the later archiving step:
- Failure before that point: reminder_mail reverts to draft (unchanged);
  sepa_mail now records a warning and continues with the remaining
  debits instead of aborting the whole batch.
- Failure after that point (Sent-copy append, moving the archive file
  into place): the reminder stays "sent" / the debit stays in the
  batch's results, an event is still logged for traceability (with an
  archive_error note and no document reference), and the caller gets a
  clear error to follow up on manually -- no rollback, no silent loss of
  the fact that the mail already went out.
2026-08-20 12:22:37 +02:00

582 lines
20 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import xml.etree.ElementTree as ET
from contextlib import contextmanager
from datetime import UTC, date, datetime
from decimal import Decimal
from email.parser import BytesParser
from email.policy import default
from ccma.domain.models import ContributionData
from ccma.services.housekeeper import Housekeeper
from ccma.services.sepa import PAIN_NAMESPACE, _safe, csv_text, pain008_bytes, pending_direct_debits
from ccma.services.sepa_mail import debit_mail_bytes, generate_debit_mails
from ccma.storage.repository import MemberRepository
from ccma.ui.sepa_dialog import _next_weekday
def _repository(tmp_path):
repository = MemberRepository(tmp_path)
repository.initialize()
member = repository.create_member(first_name="Ada", last_name="Lovelace", member_number="C3-42")
member.status = "active"
member.email = "ada@example.org"
member.account_holder = "Ada Lovelace"
member.iban = "DE89370400440532013000"
member.bic = "COBADEFFXXX"
member.mandate_reference = "MANDAT-42"
member.mandate_signed_at = "2025-01-10"
member.mandate_active = True
repository.save_member(member)
repository.save_contributions(
member.member_id,
ContributionData(
claims=[
{
"claim_id": "due",
"title": "Mitgliedsbeitrag 2026",
"amount": "150.00",
"due_date": "2026-01-31",
"status": "open",
},
{
"claim_id": "future",
"title": "Mitgliedsbeitrag 2027",
"amount": "150.00",
"due_date": "2027-01-31",
"status": "open",
},
],
payments=[{"payment_id": "payment", "amount": "25.00"}],
allocations=[{"payment_id": "payment", "claim_id": "due", "amount": "25.00"}],
),
)
return repository, member
def test_pending_debits_select_due_open_balance(tmp_path):
repository, member = _repository(tmp_path)
debits, warnings = pending_direct_debits(repository, due_until=date(2026, 12, 31))
assert warnings == []
assert len(debits) == 1
assert debits[0].member_id == member.member_id
assert debits[0].amount == Decimal("125.00")
assert debits[0].claim_ids == ("due",)
def test_pending_debits_follow_members_monthly_payment_frequency(tmp_path):
repository, member = _repository(tmp_path)
member.accepted_at = "2025-01-01"
member.membership_started_at = "2025-01-01"
member.payment_frequency = "monthly"
repository.save_member(member)
repository.save_contributions(member.member_id, ContributionData())
Housekeeper(repository).run(today=date(2026, 3, 31))
debits, warnings = pending_direct_debits(
repository,
due_from=date(2026, 1, 1),
due_until=date(2026, 3, 31),
)
assert warnings == []
assert len(debits) == 1
assert debits[0].amount == Decimal("37.50")
assert len(debits[0].claim_ids) == 3
def test_pending_debits_exclude_claims_before_lower_date(tmp_path):
repository, _member = _repository(tmp_path)
debits, warnings = pending_direct_debits(
repository,
due_from=date(2026, 2, 1),
due_until=date(2026, 12, 31),
)
assert warnings == []
assert debits == []
def test_next_weekday_moves_weekend_to_monday():
assert _next_weekday(date(2026, 8, 1)) == date(2026, 8, 3)
assert _next_weekday(date(2026, 8, 3)) == date(2026, 8, 3)
def test_sepa_text_normalizes_unsupported_characters():
assert _safe("Sébastien OConnor Müller", 70) == "Sebastien O'Connor - Müller"
def test_csv_is_semicolon_separated_and_uses_decimal_comma(tmp_path):
repository, _member = _repository(tmp_path)
debits, _warnings = pending_direct_debits(repository, due_until=date(2026, 12, 31))
text = csv_text(debits, collection_date=date(2026, 8, 3))
assert "Einzugsdatum;Mitgliedsnummer" in text
assert "125,00;EUR;MANDAT-42" in text
def test_pain008_contains_control_sum_mandate_and_creditor(tmp_path):
repository, _member = _repository(tmp_path)
debits, _warnings = pending_direct_debits(repository, due_until=date(2026, 12, 31))
organization = {
"name": "Chaos Computer Club Mannheim e.V.",
"iban": "DE89370400440532013000",
"bic": "COBADEFFXXX",
"creditor_id": "DE98ZZZ09999999999",
}
content = pain008_bytes(
debits,
collection_date=date(2026, 8, 3),
organization=organization,
message_id="CCMA-TEST",
created_at=datetime(2026, 7, 30, 12, 0, tzinfo=UTC),
)
root = ET.fromstring(content)
ns = {"p": PAIN_NAMESPACE}
assert root.findtext(".//p:GrpHdr/p:CtrlSum", namespaces=ns) == "125.00"
assert root.findtext(".//p:MndtId", namespaces=ns) == "MANDAT-42"
assert (
root.findtext(".//p:DrctDbtTx/p:CdtrSchmeId//p:Othr/p:Id", namespaces=ns)
== "DE98ZZZ09999999999"
)
assert root.find("p:CstmrDrctDbtInitn/p:PmtInf/p:CdtrSchmeId", ns) is None
assert root.findtext(".//p:ReqdColltnDt", namespaces=ns) == "2026-08-03"
def test_pain008_contains_one_logical_batch_with_multiple_transactions(tmp_path):
repository, _member = _repository(tmp_path)
debit = pending_direct_debits(repository, due_until=date(2026, 12, 31))[0][0]
organization = {
"name": "Chaos Computer Club Mannheim e.V.",
"iban": "DE89370400440532013000",
"bic": "COBADEFFXXX",
"creditor_id": "DE98ZZZ09999999999",
}
root = ET.fromstring(
pain008_bytes(
[debit, debit],
collection_date=date(2026, 8, 3),
organization=organization,
message_id="CCMA-BATCH-TEST",
)
)
ns = {"p": PAIN_NAMESPACE}
assert len(root.findall("p:CstmrDrctDbtInitn", ns)) == 1
assert len(root.findall(".//p:PmtInf", ns)) == 1
assert len(root.findall(".//p:DrctDbtTxInf", ns)) == 2
assert root.findtext(".//p:GrpHdr/p:NbOfTxs", namespaces=ns) == "2"
assert root.findtext(".//p:GrpHdr/p:CtrlSum", namespaces=ns) == "250.00"
def test_debit_mail_is_thunderbird_draft(tmp_path):
repository, member = _repository(tmp_path)
debit = pending_direct_debits(repository, due_until=date(2026, 12, 31))[0][0]
content = debit_mail_bytes(
recipient=member.email,
first_name=member.first_name,
debit=debit,
collection_date=date(2026, 8, 3),
creditor_id="DE98ZZZ09999999999",
sender_name="Verwaltung C3MA",
sender_email="verwaltung@example.org",
signature="Der Vorstand",
created_at=datetime(2026, 7, 30, 12, 0, tzinfo=UTC),
)
message = BytesParser(policy=default).parsebytes(content)
assert message["To"] == "ada@example.org"
assert str(message["X-Mozilla-Draft-Info"]).strip().startswith("internal/draft")
assert "125.00 Euro" in message.get_content()
assert "MANDAT-42" in message.get_content()
def test_generated_mail_is_exported_archived_and_logged(tmp_path):
repository, member = _repository(tmp_path)
organization = repository.get_configuration()["organization"]
organization.update(
{
"name": "Chaos Computer Club Mannheim e.V.",
"email": "verwaltung@example.org",
"iban": "DE89370400440532013000",
"bic": "COBADEFFXXX",
"creditor_id": "DE98ZZZ09999999999",
}
)
repository.save_organization(organization)
debits, _warnings = pending_direct_debits(repository, due_until=date(2026, 12, 31))
generated, warnings = generate_debit_mails(
repository,
debits,
collection_date=date(2026, 8, 3),
delivery_mode="local",
output_directory=tmp_path / "mail-export",
sender_name="Verwaltung C3MA",
sender_email="verwaltung@example.org",
signature="Der Vorstand",
)
assert warnings == []
assert generated[0].export_path.is_file()
assert generated[0].archive_path.is_file()
assert generated[0].archive_path.read_bytes() == generated[0].export_path.read_bytes()
event = repository.get_events(member.member_id)[-1]
assert event.event_type == "sepa_notification_generated"
assert event.references["document"].startswith("documents/SEPA/")
def test_generated_mail_send_mode_reuses_one_smtp_connection_for_batch(tmp_path, monkeypatch):
import ccma.services.sepa_mail as sepa_mail_module
repository, member = _repository(tmp_path)
organization = repository.get_configuration()["organization"]
organization.update(
{
"name": "Chaos Computer Club Mannheim e.V.",
"email": "verwaltung@example.org",
"iban": "DE89370400440532013000",
"bic": "COBADEFFXXX",
"creditor_id": "DE98ZZZ09999999999",
}
)
repository.save_organization(organization)
repository.save_email_settings(
delivery_mode="send",
smtp_host="mail.example.org",
smtp_port=587,
smtp_security="none",
smtp_username="",
smtp_password="",
imap_host="",
imap_port=993,
imap_security="ssl",
imap_username="",
imap_password="",
imap_drafts_folder="",
)
debits, _warnings = pending_direct_debits(repository, due_until=date(2026, 12, 31))
connections_opened = []
@contextmanager
def fake_smtp_session(settings):
connections_opened.append(settings)
yield object()
sent = []
monkeypatch.setattr(sepa_mail_module, "smtp_session", fake_smtp_session)
monkeypatch.setattr(
sepa_mail_module, "send_via_smtp", lambda client, content: sent.append(content)
)
generated, warnings = generate_debit_mails(
repository,
debits,
collection_date=date(2026, 8, 3),
delivery_mode="send",
sender_name="Verwaltung C3MA",
sender_email="verwaltung@example.org",
signature="Der Vorstand",
)
assert warnings == []
assert generated[0].export_path is None
assert generated[0].archive_path.is_file()
assert len(connections_opened) == 1, "one connection should be reused for the whole batch"
assert len(sent) == len(debits)
def test_generated_mail_send_mode_copies_to_sent_folder_reusing_one_imap_connection(
tmp_path, monkeypatch
):
import ccma.services.sepa_mail as sepa_mail_module
repository, ada = _repository(tmp_path)
second = repository.create_member(first_name="Grace", last_name="Hopper", member_number="C3-43")
second.status = "active"
second.email = "grace@example.org"
second.account_holder = "Grace Hopper"
second.iban = "DE89370400440532013000"
second.bic = "COBADEFFXXX"
second.mandate_reference = "MANDAT-43"
second.mandate_signed_at = "2025-01-10"
second.mandate_active = True
repository.save_member(second)
repository.save_contributions(
second.member_id,
ContributionData(
claims=[
{
"claim_id": "due",
"title": "Mitgliedsbeitrag 2026",
"amount": "150.00",
"due_date": "2026-01-31",
"status": "open",
}
]
),
)
organization = repository.get_configuration()["organization"]
organization.update(
{
"name": "Chaos Computer Club Mannheim e.V.",
"email": "verwaltung@example.org",
"iban": "DE89370400440532013000",
"bic": "COBADEFFXXX",
"creditor_id": "DE98ZZZ09999999999",
}
)
repository.save_organization(organization)
repository.save_email_settings(
delivery_mode="send",
smtp_host="mail.example.org",
smtp_port=587,
smtp_security="none",
smtp_username="",
smtp_password="",
imap_host="mail.example.org",
imap_port=993,
imap_security="ssl",
imap_username="",
imap_password="",
imap_drafts_folder="",
imap_sent_enabled=True,
imap_sent_folder="INBOX.Gesendet",
)
debits, _warnings = pending_direct_debits(repository, due_until=date(2026, 12, 31))
assert len(debits) == 2
monkeypatch.setattr(
sepa_mail_module, "smtp_session", contextmanager(lambda settings: iter(["smtp-client"]))
)
monkeypatch.setattr(sepa_mail_module, "send_via_smtp", lambda client, content: None)
imap_connections = []
@contextmanager
def fake_imap_session(settings):
imap_connections.append(settings)
yield "imap-client"
monkeypatch.setattr(sepa_mail_module, "imap_session", fake_imap_session)
ensured = []
appended = []
monkeypatch.setattr(
sepa_mail_module, "ensure_imap_folder", lambda client, folder: ensured.append(folder)
)
monkeypatch.setattr(
sepa_mail_module,
"append_message",
lambda client, content, *, folder, flags: appended.append((client, folder, flags)),
)
generate_debit_mails(
repository,
debits,
collection_date=date(2026, 8, 3),
delivery_mode="send",
sender_name="Verwaltung C3MA",
sender_email="verwaltung@example.org",
signature="Der Vorstand",
)
assert len(imap_connections) == 1, "one IMAP connection should be reused for the whole batch"
assert ensured == ["INBOX.Gesendet"]
assert appended == [("imap-client", "INBOX.Gesendet", r"(\Seen)")] * 2
def test_generated_mail_archiving_failure_after_smtp_success_keeps_batch_going(tmp_path, monkeypatch):
import ccma.services.sepa_mail as sepa_mail_module
repository, ada = _repository(tmp_path)
second = repository.create_member(first_name="Grace", last_name="Hopper", member_number="C3-43")
second.status = "active"
second.email = "grace@example.org"
second.account_holder = "Grace Hopper"
second.iban = "DE89370400440532013000"
second.bic = "COBADEFFXXX"
second.mandate_reference = "MANDAT-43"
second.mandate_signed_at = "2025-01-10"
second.mandate_active = True
repository.save_member(second)
repository.save_contributions(
second.member_id,
ContributionData(
claims=[
{
"claim_id": "due",
"title": "Mitgliedsbeitrag 2026",
"amount": "150.00",
"due_date": "2026-01-31",
"status": "open",
}
]
),
)
organization = repository.get_configuration()["organization"]
organization.update(
{
"name": "Chaos Computer Club Mannheim e.V.",
"email": "verwaltung@example.org",
"iban": "DE89370400440532013000",
"bic": "COBADEFFXXX",
"creditor_id": "DE98ZZZ09999999999",
}
)
repository.save_organization(organization)
repository.save_email_settings(
delivery_mode="send",
smtp_host="mail.example.org",
smtp_port=587,
smtp_security="none",
smtp_username="",
smtp_password="",
imap_host="",
imap_port=993,
imap_security="ssl",
imap_username="",
imap_password="",
imap_drafts_folder="",
)
debits, _warnings = pending_direct_debits(repository, due_until=date(2026, 12, 31))
assert len(debits) == 2
monkeypatch.setattr(
sepa_mail_module, "smtp_session", contextmanager(lambda settings: iter(["smtp-client"]))
)
sent = []
monkeypatch.setattr(sepa_mail_module, "send_via_smtp", lambda client, content: sent.append(content))
from pathlib import Path
original_write_bytes = Path.write_bytes
def _flaky_write_bytes(self, data):
if "documents" in str(self) and "C3-43" in self.name:
raise OSError("disk full")
return original_write_bytes(self, data)
monkeypatch.setattr(Path, "write_bytes", _flaky_write_bytes)
generated, warnings = generate_debit_mails(
repository,
debits,
collection_date=date(2026, 8, 3),
delivery_mode="send",
sender_name="Verwaltung C3MA",
sender_email="verwaltung@example.org",
signature="Der Vorstand",
)
# Both mails were handed to SMTP -- a broken archive write for one of them must
# not swallow the other, nor pretend the failing one was never sent.
assert len(sent) == 2
assert len(generated) == 2
assert len(warnings) == 1
assert "C3-43" in warnings[0] or "Grace" in warnings[0]
failing = next(g for g in generated if g.member_id == second.member_id)
assert not failing.archive_path.is_file()
succeeding = next(g for g in generated if g.member_id == ada.member_id)
assert succeeding.archive_path.is_file()
grace_event = repository.get_events(second.member_id)[-1]
assert grace_event.event_type == "sepa_notification_generated"
assert grace_event.data["archive_error"]
assert "document" not in grace_event.references
ada_event = repository.get_events(ada.member_id)[-1]
assert "document" in ada_event.references
def test_generated_mail_smtp_failure_for_one_debit_does_not_abort_the_batch(tmp_path, monkeypatch):
import ccma.services.sepa_mail as sepa_mail_module
from ccma.storage.repository import RepositoryError
repository, ada = _repository(tmp_path)
second = repository.create_member(first_name="Grace", last_name="Hopper", member_number="C3-43")
second.status = "active"
second.email = "grace@example.org"
second.account_holder = "Grace Hopper"
second.iban = "DE89370400440532013000"
second.bic = "COBADEFFXXX"
second.mandate_reference = "MANDAT-43"
second.mandate_signed_at = "2025-01-10"
second.mandate_active = True
repository.save_member(second)
repository.save_contributions(
second.member_id,
ContributionData(
claims=[
{
"claim_id": "due",
"title": "Mitgliedsbeitrag 2026",
"amount": "150.00",
"due_date": "2026-01-31",
"status": "open",
}
]
),
)
organization = repository.get_configuration()["organization"]
organization.update(
{
"name": "Chaos Computer Club Mannheim e.V.",
"email": "verwaltung@example.org",
"iban": "DE89370400440532013000",
"bic": "COBADEFFXXX",
"creditor_id": "DE98ZZZ09999999999",
}
)
repository.save_organization(organization)
repository.save_email_settings(
delivery_mode="send",
smtp_host="mail.example.org",
smtp_port=587,
smtp_security="none",
smtp_username="",
smtp_password="",
imap_host="",
imap_port=993,
imap_security="ssl",
imap_username="",
imap_password="",
imap_drafts_folder="",
)
debits, _warnings = pending_direct_debits(repository, due_until=date(2026, 12, 31))
assert len(debits) == 2
monkeypatch.setattr(
sepa_mail_module, "smtp_session", contextmanager(lambda settings: iter(["smtp-client"]))
)
sent = []
def _flaky_send(client, content):
if b"ada@example.org" in content:
raise RepositoryError("E-Mail konnte nicht versandt werden: connection reset")
sent.append(content)
monkeypatch.setattr(sepa_mail_module, "send_via_smtp", _flaky_send)
generated, warnings = generate_debit_mails(
repository,
debits,
collection_date=date(2026, 8, 3),
delivery_mode="send",
sender_name="Verwaltung C3MA",
sender_email="verwaltung@example.org",
signature="Der Vorstand",
)
assert len(sent) == 1
assert len(generated) == 1
assert generated[0].member_id == second.member_id
assert len(warnings) == 1
assert "connection reset" in warnings[0]
ada_events = repository.get_events(ada.member_id)
assert all(event.event_type != "sepa_notification_generated" for event in ada_events)