Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.

0 downloads

dbbasic-object-server/tests/test_object_ofx.py
Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/dbbasic-object-server.git
HTTPS
https://gitstable.com/git/askrobots/dbbasic-object-server.git
SSH
ssh://git@gitstable.com:2222/askrobots/dbbasic-object-server.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

danq9 signup: the person chooses the password and name on the confirm page 83c7aa6 2 weeks, 5 days ago History
test_object_ofx.py · 14.9 KB · 467 lines
1"""OFX/QFX statement parsing (plan/bank-import-reconciliation-spec.md,
2"v1.5: action_import_ofx").
3
4The tests are the simulated bank feed, in both dialects that hide under one
5file extension: an OFX 1.x SGML export with the unclosed leaf tags real
6banks actually produce, and an OFX 2.x XML export of the identical
7statement, pinned to produce the identical canonical shape. What matters
8here is not "does an XML-ish string parse" but: dates are taken as printed
9(never timezone-shifted), amounts are exact integer cents via Decimal,
10FITID becomes the strong external_id, LEDGERBAL absence is None (not 0),
11and a hostile DOCTYPE/ENTITY is refused before either parser ever runs.
12"""
13
14import pytest
15
16import object_banking
17import object_ofx
18from object_ofx import OfxError
19
20
21# --- fixtures -----------------------------------------------------------------
22#
23# The SGML statement is deliberately written the way real OFX 1.x exports
24# look: header block, blank line, then leaf tags with no closing tag
25# (</DTPOSTED> etc. never appear). The XML statement carries the same two
26# transactions and the same balance, fully closed, to pin that both
27# dialects land in the same canonical shape.
28
29SGML_STATEMENT = """OFXHEADER:100
30DATA:OFXSGML
31VERSION:102
32SECURITY:NONE
33ENCODING:USASCII
34CHARSET:1252
35COMPRESSION:NONE
36OLDFILEUID:NONE
37NEWFILEUID:NONE
38
39<OFX>
40<SIGNONMSGSRSV1>
41<SONRS>
42<STATUS>
43<CODE>0
44<SEVERITY>INFO
45</STATUS>
46<DTSERVER>20260710120000[-5:EST]
47<LANGUAGE>ENG
48</SONRS>
49</SIGNONMSGSRSV1>
50<BANKMSGSRSV1>
51<STMTTRNRS>
52<TRNUID>1
53<STATUS>
54<CODE>0
55<SEVERITY>INFO
56</STATUS>
57<STMTRS>
58<CURDEF>USD
59<BANKACCTFROM>
60<BANKID>123456789
61<ACCTID>987654321
62<ACCTTYPE>CHECKING
63</BANKACCTFROM>
64<BANKTRANLIST>
65<DTSTART>20260701000000[-5:EST]
66<DTEND>20260710000000[-5:EST]
67<STMTTRN>
68<TRNTYPE>CREDIT
69<DTPOSTED>20260702120000[-5:EST]
70<TRNAMT>1500.00
71<FITID>2026070200001
72<NAME>ACME CORP PAYMENT
73<MEMO>INV-1001
74</STMTTRN>
75<STMTTRN>
76<TRNTYPE>DEBIT
77<DTPOSTED>20260705
78<TRNAMT>-25.00
79<FITID>2026070500001
80<NAME>MONTHLY SERVICE FEE
81</STMTTRN>
82</BANKTRANLIST>
83<LEDGERBAL>
84<BALAMT>1475.00
85<DTASOF>20260710120000[-5:EST]
86</LEDGERBAL>
87</STMTRS>
88</STMTTRNRS>
89</BANKMSGSRSV1>
90</OFX>
91"""
92
93XML_STATEMENT = """<?xml version="1.0" encoding="UTF-8"?>
94<?OFX OFXHEADER="200" VERSION="200" SECURITY="NONE" OLDFILEUID="NONE" NEWFILEUID="NONE"?>
95<OFX>
96 <SIGNONMSGSRSV1>
97 <SONRS>
98 <STATUS><CODE>0</CODE><SEVERITY>INFO</SEVERITY></STATUS>
99 <DTSERVER>20260710120000</DTSERVER>
100 <LANGUAGE>ENG</LANGUAGE>
101 </SONRS>
102 </SIGNONMSGSRSV1>
103 <BANKMSGSRSV1>
104 <STMTTRNRS>
105 <TRNUID>1</TRNUID>
106 <STATUS><CODE>0</CODE><SEVERITY>INFO</SEVERITY></STATUS>
107 <STMTRS>
108 <CURDEF>USD</CURDEF>
109 <BANKACCTFROM>
110 <BANKID>123456789</BANKID>
111 <ACCTID>987654321</ACCTID>
112 <ACCTTYPE>CHECKING</ACCTTYPE>
113 </BANKACCTFROM>
114 <BANKTRANLIST>
115 <DTSTART>20260701000000</DTSTART>
116 <DTEND>20260710000000</DTEND>
117 <STMTTRN>
118 <TRNTYPE>CREDIT</TRNTYPE>
119 <DTPOSTED>20260702120000[-5:EST]</DTPOSTED>
120 <TRNAMT>1500.00</TRNAMT>
121 <FITID>2026070200001</FITID>
122 <NAME>ACME CORP PAYMENT</NAME>
123 <MEMO>INV-1001</MEMO>
124 </STMTTRN>
125 <STMTTRN>
126 <TRNTYPE>DEBIT</TRNTYPE>
127 <DTPOSTED>20260705</DTPOSTED>
128 <TRNAMT>-25.00</TRNAMT>
129 <FITID>2026070500001</FITID>
130 <NAME>MONTHLY SERVICE FEE</NAME>
131 </STMTTRN>
132 </BANKTRANLIST>
133 <LEDGERBAL>
134 <BALAMT>1475.00</BALAMT>
135 <DTASOF>20260710120000</DTASOF>
136 </LEDGERBAL>
137 </STMTRS>
138 </STMTTRNRS>
139 </BANKMSGSRSV1>
140</OFX>
141"""
142
143# Same statement, minus the LEDGERBAL block entirely -- a bank that gives
144# no balances at all (object_banking.py's tie_out "ran: False" case).
145SGML_NO_LEDGERBAL = SGML_STATEMENT.replace(
146 """<LEDGERBAL>
147<BALAMT>1475.00
148<DTASOF>20260710120000[-5:EST]
149</LEDGERBAL>
150""", "")
151
152CC_STATEMENT = """OFXHEADER:100
153DATA:OFXSGML
154VERSION:102
155SECURITY:NONE
156ENCODING:USASCII
157CHARSET:1252
158COMPRESSION:NONE
159OLDFILEUID:NONE
160NEWFILEUID:NONE
161
162<OFX>
163<CREDITCARDMSGSRSV1>
164<CCSTMTTRNRS>
165<TRNUID>1
166<STATUS>
167<CODE>0
168<SEVERITY>INFO
169</STATUS>
170<CCSTMTRS>
171<CURDEF>USD
172<CCACCTFROM>
173<ACCTID>4111111111111111
174</CCACCTFROM>
175<BANKTRANLIST>
176<DTSTART>20260701000000
177<DTEND>20260710000000
178<STMTTRN>
179<TRNTYPE>DEBIT
180<DTPOSTED>20260703
181<TRNAMT>-89.99
182<FITID>CC0001
183<NAME>ONLINE STORE
184</STMTTRN>
185</BANKTRANLIST>
186<LEDGERBAL>
187<BALAMT>-89.99
188<DTASOF>20260710
189</LEDGERBAL>
190</CCSTMTRS>
191</CCSTMTTRNRS>
192</CREDITCARDMSGSRSV1>
193</OFX>
194"""
195
196
197def _min_sgml(stmttrn_body: str) -> str:
198 """A minimal single-transaction OFX 1.x envelope, for pinning one
199 behaviour (a date form, a NAME/MEMO combination) without repeating the
200 whole header/account boilerplate each time."""
201 return _min_sgml_multi(f"<STMTTRN>\n{stmttrn_body}\n</STMTTRN>")
202
203
204def _min_sgml_multi(stmttrn_blocks: str) -> str:
205 """Like _min_sgml, but takes one or more already-wrapped
206 <STMTTRN>...</STMTTRN> blocks verbatim -- for tests that need more
207 than one transaction in the same BANKTRANLIST (same-day twins)."""
208 return f"""OFXHEADER:100
209DATA:OFXSGML
210VERSION:102
211SECURITY:NONE
212ENCODING:USASCII
213CHARSET:1252
214COMPRESSION:NONE
215OLDFILEUID:NONE
216NEWFILEUID:NONE
217
218<OFX>
219<BANKMSGSRSV1>
220<STMTTRNRS>
221<STMTRS>
222<CURDEF>USD
223<BANKACCTFROM>
224<BANKID>123456789
225<ACCTID>987654321
226<ACCTTYPE>CHECKING
227</BANKACCTFROM>
228<BANKTRANLIST>
229{stmttrn_blocks}
230</BANKTRANLIST>
231</STMTRS>
232</STMTTRNRS>
233</BANKMSGSRSV1>
234</OFX>
235"""
236
237
238def _strip_raw(lines):
239 return [{k: v for k, v in line.items() if k != "raw"} for line in lines]
240
241
242# --- 1. OFX 1.x SGML with unclosed tags ---------------------------------------
243
244def test_sgml_statement_with_unclosed_tags_parses():
245 result = object_ofx.parse_ofx(SGML_STATEMENT)
246 lines = result["lines"]
247 assert len(lines) == 2
248 assert [l["posted_on"] for l in lines] == ["2026-07-02", "2026-07-05"]
249 assert [l["amount_cents"] for l in lines] == [150000, -2500]
250 assert result["account"] == {
251 "account_id": "987654321", "bank_id": "123456789", "account_type": "CHECKING"}
252 assert result["currency"] == "USD"
253 # raw keeps the original transaction block verbatim -- the evidence.
254 assert "1500.00" in lines[0]["raw"] and "ACME CORP" in lines[0]["raw"]
255
256
257# --- 2. OFX 2.x XML parses to the same shape ----------------------------------
258
259def test_xml_statement_parses_to_the_same_shape_as_sgml():
260 sgml = object_ofx.parse_ofx(SGML_STATEMENT)
261 xml = object_ofx.parse_ofx(XML_STATEMENT)
262 assert _strip_raw(sgml["lines"]) == _strip_raw(xml["lines"])
263 assert sgml["account"] == xml["account"]
264 assert sgml["balances"] == xml["balances"]
265 assert sgml["period"] == xml["period"]
266 assert sgml["currency"] == xml["currency"]
267 # XML's raw is a real closed-tag slice, distinct text from SGML's.
268 assert "<TRNAMT>1500.00</TRNAMT>" in xml["lines"][0]["raw"]
269
270
271def test_is_ofx_sniffs_both_dialects_and_rejects_garbage():
272 assert object_ofx.is_ofx(SGML_STATEMENT) is True
273 assert object_ofx.is_ofx(XML_STATEMENT) is True
274 assert object_ofx.is_ofx("just some random text, not a statement") is False
275 assert object_ofx.is_ofx("") is False
276
277
278# --- 3. signed amounts, exact cents --------------------------------------------
279
280def test_signed_amounts_to_exact_integer_cents():
281 assert object_ofx.parse_ofx_amount("-25.00") == -2500
282 assert object_ofx.parse_ofx_amount("1500.00") == 150000
283 assert object_ofx.parse_ofx_amount("+42.10") == 4210
284 assert object_ofx.parse_ofx_amount("0.00") == 0
285 with pytest.raises(OfxError):
286 object_ofx.parse_ofx_amount("")
287 with pytest.raises(OfxError):
288 object_ofx.parse_ofx_amount("not-a-number")
289
290
291def test_deposit_positive_and_withdrawal_negative_in_a_real_statement():
292 lines = object_ofx.parse_ofx(SGML_STATEMENT)["lines"]
293 deposit, withdrawal = lines
294 assert deposit["amount_cents"] == 150000
295 assert withdrawal["amount_cents"] == -2500
296
297
298# --- 4. FITID becomes external_id ---------------------------------------------
299
300def test_fitid_becomes_external_id():
301 lines = object_ofx.parse_ofx(SGML_STATEMENT)["lines"]
302 assert lines[0]["external_id"] == "2026070200001"
303 assert lines[1]["external_id"] == "2026070500001"
304
305
306# --- 5. dates: full timestamp+tz and bare YYYYMMDD, unshifted -----------------
307
308def test_full_timestamp_with_negative_tz_is_taken_as_printed():
309 fixture = _min_sgml(
310 "<DTPOSTED>20260702120000[-5:EST]\n<TRNAMT>10.00\n<FITID>X1\n<NAME>A")
311 assert object_ofx.parse_ofx(fixture)["lines"][0]["posted_on"] == "2026-07-02"
312
313
314def test_bare_yyyymmdd_date_parses_to_the_same_calendar_date():
315 fixture = _min_sgml("<DTPOSTED>20260705\n<TRNAMT>10.00\n<FITID>X2\n<NAME>A")
316 assert object_ofx.parse_ofx(fixture)["lines"][0]["posted_on"] == "2026-07-05"
317
318
319def test_extreme_positive_tz_offset_near_midnight_does_not_shift_the_day():
320 # A +13 offset on a midnight timestamp is exactly the case a naive
321 # "convert to UTC" implementation would slide into the previous day.
322 fixture = _min_sgml(
323 "<DTPOSTED>20260101000000.500[+13:NZDT]\n<TRNAMT>10.00\n<FITID>X3\n<NAME>A")
324 assert object_ofx.parse_ofx(fixture)["lines"][0]["posted_on"] == "2026-01-01"
325
326
327def test_parse_ofx_date_rejects_unparseable_text():
328 with pytest.raises(OfxError):
329 object_ofx.parse_ofx_date("not-a-date")
330 with pytest.raises(OfxError):
331 object_ofx.parse_ofx_date("")
332
333
334# --- 6. NAME/MEMO combination --------------------------------------------------
335
336def test_name_and_memo_combine_when_both_present_and_different():
337 fixture = _min_sgml(
338 "<DTPOSTED>20260702\n<TRNAMT>10.00\n<FITID>X1\n"
339 "<NAME>ACME CORP\n<MEMO>INV-1001")
340 assert object_ofx.parse_ofx(fixture)["lines"][0]["description"] == "ACME CORP — INV-1001"
341
342
343def test_name_alone_is_used_as_is():
344 fixture = _min_sgml("<DTPOSTED>20260702\n<TRNAMT>10.00\n<FITID>X2\n<NAME>ACME CORP")
345 assert object_ofx.parse_ofx(fixture)["lines"][0]["description"] == "ACME CORP"
346
347
348def test_identical_name_and_memo_are_not_duplicated():
349 fixture = _min_sgml(
350 "<DTPOSTED>20260702\n<TRNAMT>10.00\n<FITID>X3\n"
351 "<NAME>ACME CORP\n<MEMO>ACME CORP")
352 assert object_ofx.parse_ofx(fixture)["lines"][0]["description"] == "ACME CORP"
353
354
355def test_neither_name_nor_memo_is_tolerated_as_empty_description():
356 fixture = _min_sgml("<DTPOSTED>20260702\n<TRNAMT>10.00\n<FITID>X4")
357 result = object_ofx.parse_ofx(fixture)
358 assert result["lines"][0]["description"] == ""
359
360
361# --- 7. LEDGERBAL -> closing balance; absent -> None, not 0 ------------------
362
363def test_ledgerbal_becomes_closing_balance_cents():
364 balances = object_ofx.parse_ofx(SGML_STATEMENT)["balances"]
365 assert balances["closing_balance_cents"] == 147500
366 assert balances["as_of"] == "2026-07-10"
367 # No standard OFX tag carries an opening balance -- honestly None.
368 assert balances["opening_balance_cents"] is None
369
370
371def test_missing_ledgerbal_yields_none_not_zero():
372 balances = object_ofx.parse_ofx(SGML_NO_LEDGERBAL)["balances"]
373 assert balances["closing_balance_cents"] is None
374 assert balances["as_of"] == ""
375
376
377# --- 8. credit-card (CCSTMTRS) statements --------------------------------------
378
379def test_credit_card_statement_parses():
380 result = object_ofx.parse_ofx(CC_STATEMENT)
381 assert result["account"] == {
382 "account_id": "4111111111111111", "bank_id": "", "account_type": "CREDITCARD"}
383 assert len(result["lines"]) == 1
384 line = result["lines"][0]
385 assert line["posted_on"] == "2026-07-03"
386 assert line["amount_cents"] == -8999
387 assert line["external_id"] == "CC0001"
388 assert result["balances"]["closing_balance_cents"] == -8999
389
390
391# --- 9. malformed input refused; DOCTYPE/ENTITY refused ------------------------
392
393def test_garbage_input_raises_ofxerror():
394 with pytest.raises(OfxError):
395 object_ofx.parse_ofx("this is not an OFX file, just some prose.\n")
396 with pytest.raises(OfxError):
397 object_ofx.parse_ofx("")
398
399
400def test_truncated_xml_raises_ofxerror_not_a_parser_traceback():
401 truncated = XML_STATEMENT.replace("</OFX>", "") # unterminated root
402 with pytest.raises(OfxError):
403 object_ofx.parse_ofx(truncated)
404
405
406def test_stmttrn_missing_dtposted_is_refused_not_half_read():
407 fixture = _min_sgml("<TRNAMT>10.00\n<FITID>X5\n<NAME>NO DATE HERE")
408 with pytest.raises(OfxError):
409 object_ofx.parse_ofx(fixture)
410
411
412def test_doctype_declaration_is_refused_before_parsing():
413 attack = ('<?xml version="1.0"?>\n'
414 '<!DOCTYPE ofx [<!ENTITY xxe SYSTEM "file:///etc/passwd">]>\n'
415 + XML_STATEMENT)
416 with pytest.raises(OfxError):
417 object_ofx.parse_ofx(attack)
418
419
420def test_entity_declaration_without_doctype_wrapper_is_also_refused():
421 # Belt and suspenders: refuse on the ENTITY token itself, not only when
422 # it is nested inside a DOCTYPE block we happen to recognise.
423 attack = "<!ENTITY xxe SYSTEM \"file:///etc/passwd\">\n" + SGML_STATEMENT
424 with pytest.raises(OfxError):
425 object_ofx.parse_ofx(attack)
426
427
428def test_doctype_attack_on_the_sgml_dialect_is_also_refused():
429 # The SGML branch never touches an XML parser at all, so it is not
430 # actually vulnerable to entity expansion -- but a hostile DOCTYPE has
431 # no legitimate place in a bank statement of either dialect, and the
432 # refusal is applied uniformly before dialect detection even runs.
433 attack = "<!DOCTYPE ofx [<!ENTITY xxe \"boom\">]>\n" + SGML_STATEMENT
434 with pytest.raises(OfxError):
435 object_ofx.parse_ofx(attack)
436
437
438# --- 10. round-trip through object_banking.assign_line_hashes ----------------
439
440def test_parsed_lines_get_stable_line_hashes_through_object_banking():
441 parsed = object_ofx.parse_ofx(SGML_STATEMENT)
442 stamped = object_banking.assign_line_hashes(parsed["lines"])
443 assert all(line["line_hash"] for line in stamped)
444
445 # Deterministic: hashing the same parsed lines again gives identical
446 # hashes -- this is what makes file-level re-import a safe no-op.
447 stamped_again = object_banking.assign_line_hashes(parsed["lines"])
448 assert [l["line_hash"] for l in stamped] == [l["line_hash"] for l in stamped_again]
449
450 # Pins the actual formula, not just "truthy": ordinal 0 for a line
451 # with no same-day twin in this statement.
452 first = stamped[0]
453 expected = object_banking.line_hash(
454 first["posted_on"], first["amount_cents"], first["description"], 0)
455 assert first["line_hash"] == expected
456
457
458def test_same_day_twin_lines_get_disambiguated_ordinals():
459 # Two genuine same-day twins (identical date/amount/description, the
460 # bank's own distinct FITIDs) in one BANKTRANLIST.
461 twins = ("<STMTTRN>\n<DTPOSTED>20260702\n<TRNAMT>-4.50\n<FITID>T1\n<NAME>COFFEE\n</STMTTRN>\n"
462 "<STMTTRN>\n<DTPOSTED>20260702\n<TRNAMT>-4.50\n<FITID>T2\n<NAME>COFFEE\n</STMTTRN>")
463 parsed = object_ofx.parse_ofx(_min_sgml_multi(twins))
464 assert len(parsed["lines"]) == 2
465 stamped = object_banking.assign_line_hashes(parsed["lines"])
466 assert stamped[0]["line_hash"] != stamped[1]["line_hash"] # twins, still distinct
467 assert stamped[0]["external_id"] != stamped[1]["external_id"] # FITID already distinguishes them