{ "componentChunkName": "component---src-templates-course-content-template-js", "path": "/part-5/5-in-memory-analysis-pipeline/", "result": {"data":{"page":{"htmlAst":{"type":"element","tagName":"div","properties":{},"children":[{"type":"element","tagName":"text-box","properties":{"variant":"learningObjectives","name":"Learning objectives"},"children":[{"type":"element","tagName":"p","properties":{},"children":[{"type":"text","value":"After this section"}]},{"type":"element","tagName":"ul","properties":{},"children":[{"type":"text","value":"\n"},{"type":"element","tagName":"li","properties":{},"children":[{"type":"text","value":"You will be able to trace state which changes as a sequence is processed"}]},{"type":"text","value":"\n"},{"type":"element","tagName":"li","properties":{},"children":[{"type":"text","value":"You will be able to validate, transform, summarize and present in-memory records"}]},{"type":"text","value":"\n"},{"type":"element","tagName":"li","properties":{},"children":[{"type":"text","value":"You will be able to inspect an agent-written pipeline one stage at a time"}]},{"type":"text","value":"\n"}]}]},{"type":"element","tagName":"programming-exercise","properties":{"name":"Longest repetition in melody","tmcname":"part05-27_longest_repetition"},"children":[{"type":"element","tagName":"p","properties":{},"children":[{"type":"element","tagName":"strong","properties":{},"children":[{"type":"text","value":"Mode: Code review and repair"}]}]},{"type":"element","tagName":"p","properties":{},"children":[{"type":"text","value":"A melody can be represented as a string of note names. A repetition is a consecutive sequence of the same note. For example, "},{"type":"element","tagName":"code","properties":{"className":["language-text"]},"children":[{"type":"text","value":"\"cccedddfeeddc\""}]},{"type":"text","value":" contains repetitions of three "},{"type":"element","tagName":"code","properties":{"className":["language-text"]},"children":[{"type":"text","value":"c"}]},{"type":"text","value":" notes and three "},{"type":"element","tagName":"code","properties":{"className":["language-text"]},"children":[{"type":"text","value":"d"}]},{"type":"text","value":" notes."}]},{"type":"element","tagName":"p","properties":{},"children":[{"type":"text","value":"The starter's "},{"type":"element","tagName":"code","properties":{"className":["language-text"]},"children":[{"type":"text","value":"longest_repetition(melody: str)"}]},{"type":"text","value":" function updates the longest run only when the note changes. Predict its results for "},{"type":"element","tagName":"code","properties":{"className":["language-text"]},"children":[{"type":"text","value":"\"aaabbc\""}]},{"type":"text","value":" and "},{"type":"element","tagName":"code","properties":{"className":["language-text"]},"children":[{"type":"text","value":"\"abccc\""}]},{"type":"text","value":", then run both calls. Repair the function so that a longest run at the end of the melody is also finalized."}]},{"type":"element","tagName":"div","properties":{"className":["gatsby-highlight"],"dataLanguage":"python"},"children":[{"type":"element","tagName":"pre","properties":{"className":["language-python"]},"children":[{"type":"element","tagName":"code","properties":{"className":["language-python"]},"children":[{"type":"element","tagName":"span","properties":{"className":["token","keyword"]},"children":[{"type":"text","value":"print"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"("}]},{"type":"text","value":"longest_repetition"},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"("}]},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"\"cde\""}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":")"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":")"}]},{"type":"text","value":"\n"},{"type":"element","tagName":"span","properties":{"className":["token","keyword"]},"children":[{"type":"text","value":"print"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"("}]},{"type":"text","value":"longest_repetition"},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"("}]},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"\"cccedddfeeddc\""}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":")"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":")"}]},{"type":"text","value":"\n"},{"type":"element","tagName":"span","properties":{"className":["token","keyword"]},"children":[{"type":"text","value":"print"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"("}]},{"type":"text","value":"longest_repetition"},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"("}]},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"\"gggggggg\""}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":")"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":")"}]}]}]}]},{"type":"element","tagName":"sample-output","properties":{},"children":[{"type":"element","tagName":"p","properties":{},"children":[{"type":"text","value":"1\n3\n8"}]}]},{"type":"element","tagName":"p","properties":{},"children":[{"type":"text","value":"You may assume the melody contains at least one note."}]}]},{"type":"element","tagName":"h2","properties":{"id":"from-records-to-a-result","style":"position:relative;"},"children":[{"type":"text","value":"From records to a result"},{"type":"element","tagName":"a","properties":{"href":"#from-records-to-a-result","ariaLabel":"from records to a result permalink","className":["anchor","after"]},"children":[{"type":"element","tagName":"svg","properties":{"ariaHidden":"true","focusable":"false","height":"16","version":"1.1","viewBox":"0 0 16 16","width":"16"},"children":[{"type":"element","tagName":"path","properties":{"fillRule":"evenodd","d":"M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"},"children":[]}]}]}]},{"type":"element","tagName":"p","properties":{},"children":[{"type":"text","value":"A small analysis pipeline has distinct stages even when all its data is already in memory. Using the vocabulary from "},{"type":"element","tagName":"a","properties":{"href":"/programming-digital-humanities/part-5/0-working-with-ai#pipeline-stage-labels"},"children":[{"type":"text","value":"the start of this part"}]},{"type":"text","value":":"}]},{"type":"element","tagName":"ol","properties":{},"children":[{"type":"text","value":"\n"},{"type":"element","tagName":"li","properties":{},"children":[{"type":"element","tagName":"strong","properties":{},"children":[{"type":"text","value":"Validate:"}]},{"type":"text","value":" decide which input records satisfy the data contract."}]},{"type":"text","value":"\n"},{"type":"element","tagName":"li","properties":{},"children":[{"type":"element","tagName":"strong","properties":{},"children":[{"type":"text","value":"Transform and summarize:"}]},{"type":"text","value":" place values into a useful result shape and combine repeated observations."}]},{"type":"text","value":"\n"},{"type":"element","tagName":"li","properties":{},"children":[{"type":"element","tagName":"strong","properties":{},"children":[{"type":"text","value":"Present:"}]},{"type":"text","value":" turn the structured summary into consistently ordered rows."}]},{"type":"text","value":"\n"}]},{"type":"element","tagName":"p","properties":{},"children":[{"type":"text","value":"Keeping these stages separate makes intermediate values easy to inspect. It also gives an agent a clearer specification than a single request to “analyse this data”."}]},{"type":"element","tagName":"programming-exercise","properties":{"name":"Collection summary","tmcname":"part05-28_collection_summary"},"children":[{"type":"element","tagName":"p","properties":{},"children":[{"type":"element","tagName":"strong","properties":{},"children":[{"type":"text","value":"Pipeline stage(s): Validate, Transform, Summarize, Present"}]}]},{"type":"element","tagName":"p","properties":{},"children":[{"type":"element","tagName":"strong","properties":{},"children":[{"type":"text","value":"Mode: Agent-assisted change"}]}]},{"type":"element","tagName":"p","properties":{},"children":[{"type":"text","value":"Build a pipeline for an in-memory list of collection records. A valid record has exactly the required usable information:"}]},{"type":"element","tagName":"ul","properties":{},"children":[{"type":"text","value":"\n"},{"type":"element","tagName":"li","properties":{},"children":[{"type":"element","tagName":"code","properties":{"className":["language-text"]},"children":[{"type":"text","value":"collection"}]},{"type":"text","value":": a non-empty string;"}]},{"type":"text","value":"\n"},{"type":"element","tagName":"li","properties":{},"children":[{"type":"element","tagName":"code","properties":{"className":["language-text"]},"children":[{"type":"text","value":"language"}]},{"type":"text","value":": a non-empty string; and"}]},{"type":"text","value":"\n"},{"type":"element","tagName":"li","properties":{},"children":[{"type":"element","tagName":"code","properties":{"className":["language-text"]},"children":[{"type":"text","value":"documents"}]},{"type":"text","value":": a non-negative integer."}]},{"type":"text","value":"\n"}]},{"type":"element","tagName":"p","properties":{},"children":[{"type":"text","value":"Extra fields are allowed. Invalid records should be skipped. Use boolean checks; exception handling is not needed in this exercise."}]},{"type":"element","tagName":"p","properties":{},"children":[{"type":"text","value":"Implement these functions:"}]},{"type":"element","tagName":"ul","properties":{},"children":[{"type":"text","value":"\n"},{"type":"element","tagName":"li","properties":{},"children":[{"type":"element","tagName":"code","properties":{"className":["language-text"]},"children":[{"type":"text","value":"valid_record(record: dict) -> bool"}]},{"type":"text","value":" checks the contract above."}]},{"type":"text","value":"\n"},{"type":"element","tagName":"li","properties":{},"children":[{"type":"element","tagName":"code","properties":{"className":["language-text"]},"children":[{"type":"text","value":"collection_summary(records: list) -> dict"}]},{"type":"text","value":" returns the shape "},{"type":"element","tagName":"code","properties":{"className":["language-text"]},"children":[{"type":"text","value":"collection -> language -> total documents"}]},{"type":"text","value":". Repeated collection/language observations must be added together."}]},{"type":"text","value":"\n"},{"type":"element","tagName":"li","properties":{},"children":[{"type":"element","tagName":"code","properties":{"className":["language-text"]},"children":[{"type":"text","value":"format_summary(summary: dict) -> list"}]},{"type":"text","value":" returns strings in the format "},{"type":"element","tagName":"code","properties":{"className":["language-text"]},"children":[{"type":"text","value":"collection | language | documents"}]},{"type":"text","value":", sorted first by collection and then by language."}]},{"type":"text","value":"\n"}]},{"type":"element","tagName":"p","properties":{},"children":[{"type":"text","value":"For example:"}]},{"type":"element","tagName":"div","properties":{"className":["gatsby-highlight"],"dataLanguage":"python"},"children":[{"type":"element","tagName":"pre","properties":{"className":["language-python"]},"children":[{"type":"element","tagName":"code","properties":{"className":["language-python"]},"children":[{"type":"text","value":"records "},{"type":"element","tagName":"span","properties":{"className":["token","operator"]},"children":[{"type":"text","value":"="}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"["}]},{"type":"text","value":"\n "},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"{"}]},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"\"collection\""}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":":"}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"\"Letters\""}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"\"language\""}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":":"}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"\"English\""}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"\"documents\""}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":":"}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"12"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"}"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":"\n "},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"{"}]},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"\"collection\""}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":":"}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"\"Maps\""}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"\"language\""}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":":"}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"\"Finnish\""}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"\"documents\""}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":":"}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"4"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"}"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":"\n "},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"{"}]},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"\"collection\""}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":":"}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"\"Letters\""}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"\"language\""}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":":"}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"\"English\""}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"\"documents\""}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":":"}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"3"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"}"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":"\n "},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"{"}]},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"\"collection\""}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":":"}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"\"Letters\""}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"\"language\""}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":":"}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"\"French\""}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"\"documents\""}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":":"}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"5"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"}"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":"\n "},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"{"}]},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"\"collection\""}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":":"}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"\"Incomplete\""}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":","}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","string"]},"children":[{"type":"text","value":"\"documents\""}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":":"}]},{"type":"text","value":" "},{"type":"element","tagName":"span","properties":{"className":["token","number"]},"children":[{"type":"text","value":"20"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"}"}]},{"type":"text","value":"\n"},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"]"}]},{"type":"text","value":"\n\nsummary "},{"type":"element","tagName":"span","properties":{"className":["token","operator"]},"children":[{"type":"text","value":"="}]},{"type":"text","value":" collection_summary"},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"("}]},{"type":"text","value":"records"},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":")"}]},{"type":"text","value":"\n"},{"type":"element","tagName":"span","properties":{"className":["token","keyword"]},"children":[{"type":"text","value":"for"}]},{"type":"text","value":" row "},{"type":"element","tagName":"span","properties":{"className":["token","keyword"]},"children":[{"type":"text","value":"in"}]},{"type":"text","value":" format_summary"},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"("}]},{"type":"text","value":"summary"},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":")"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":":"}]},{"type":"text","value":"\n "},{"type":"element","tagName":"span","properties":{"className":["token","keyword"]},"children":[{"type":"text","value":"print"}]},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":"("}]},{"type":"text","value":"row"},{"type":"element","tagName":"span","properties":{"className":["token","punctuation"]},"children":[{"type":"text","value":")"}]}]}]}]},{"type":"element","tagName":"sample-output","properties":{},"children":[{"type":"element","tagName":"p","properties":{},"children":[{"type":"text","value":"Letters | English | 15\nLetters | French | 5\nMaps | Finnish | 4"}]}]},{"type":"element","tagName":"p","properties":{},"children":[{"type":"text","value":"Follow the agent-assisted workflow:"}]},{"type":"element","tagName":"ol","properties":{},"children":[{"type":"text","value":"\n"},{"type":"element","tagName":"li","properties":{},"children":[{"type":"text","value":"Hand-calculate the expected summary for the fixture above and write down each function's input and output shape."}]},{"type":"text","value":"\n"},{"type":"element","tagName":"li","properties":{},"children":[{"type":"text","value":"Ask an agent to draft the three-stage pipeline using only core Python."}]},{"type":"text","value":"\n"},{"type":"element","tagName":"li","properties":{},"children":[{"type":"text","value":"Inspect each stage separately. In particular, find the condition which rejects an invalid record and the update which accumulates rather than overwrites a repeated observation."}]},{"type":"text","value":"\n"},{"type":"element","tagName":"li","properties":{},"children":[{"type":"text","value":"Run assertions for each function. Then personally add a second collection with two languages and verify the sorted rows."}]},{"type":"text","value":"\n"}]}]}]},"html":"

After this section

Mode: Code review and repair

A melody can be represented as a string of note names. A repetition is a consecutive sequence of the same note. For example, \"cccedddfeeddc\" contains repetitions of three c notes and three d notes.

The starter's longest_repetition(melody: str) function updates the longest run only when the note changes. Predict its results for \"aaabbc\" and \"abccc\", then run both calls. Repair the function so that a longest run at the end of the melody is also finalized.

print(longest_repetition(\"cde\"))\nprint(longest_repetition(\"cccedddfeeddc\"))\nprint(longest_repetition(\"gggggggg\"))

1\n3\n8

You may assume the melody contains at least one note.

From records to a result

A small analysis pipeline has distinct stages even when all its data is already in memory. Using the vocabulary from the start of this part:

    \n
  1. Validate: decide which input records satisfy the data contract.
    2. \n
  2. Transform and summarize: place values into a useful result shape and combine repeated observations.
    3. \n
  3. Present: turn the structured summary into consistently ordered rows.
    4. \n

Keeping these stages separate makes intermediate values easy to inspect. It also gives an agent a clearer specification than a single request to “analyse this data”.

Pipeline stage(s): Validate, Transform, Summarize, Present

Mode: Agent-assisted change

Build a pipeline for an in-memory list of collection records. A valid record has exactly the required usable information:

Extra fields are allowed. Invalid records should be skipped. Use boolean checks; exception handling is not needed in this exercise.

Implement these functions:

For example:

records = [\n    {\"collection\": \"Letters\", \"language\": \"English\", \"documents\": 12},\n    {\"collection\": \"Maps\", \"language\": \"Finnish\", \"documents\": 4},\n    {\"collection\": \"Letters\", \"language\": \"English\", \"documents\": 3},\n    {\"collection\": \"Letters\", \"language\": \"French\", \"documents\": 5},\n    {\"collection\": \"Incomplete\", \"documents\": 20}\n]\n\nsummary = collection_summary(records)\nfor row in format_summary(summary):\n    print(row)

Letters | English | 15\nLetters | French | 5\nMaps | Finnish | 4

Follow the agent-assisted workflow:

    \n
  1. Hand-calculate the expected summary for the fixture above and write down each function's input and output shape.
    2. \n
  2. Ask an agent to draft the three-stage pipeline using only core Python.
    3. \n
  3. Inspect each stage separately. In particular, find the condition which rejects an invalid record and the update which accumulates rather than overwrites a repeated observation.
    4. \n
  4. Run assertions for each function. Then personally add a second collection with two languages and verify the sorted rows.
    5. \n
","frontmatter":{"path":"/part-5/5-in-memory-analysis-pipeline/","title":"An in-memory analysis pipeline"},"fileAbsolutePath":"/home/runner/work/programming-digital-humanities/programming-digital-humanities/data/part-5/5-in-memory-analysis-pipeline.md"},"allPages":{"edges":[{"node":{"id":"5d899565-03f0-55ea-8073-4ee194950c4d","frontmatter":{"path":"/ai-tutor/","title":"Using the AI tutor on the course"}}},{"node":{"id":"c6198176-c061-5cca-a182-cc80cdb14e38","frontmatter":{"path":"/all-exercises/","title":"All exercises"}}},{"node":{"id":"6c2bb78f-1497-5d72-a964-8cb55e47559e","frontmatter":{"path":"/error_messages/","title":"Common error messages"}}},{"node":{"id":"96430bcf-4526-5d85-9206-c7336a6be270","frontmatter":{"path":"/faq/","title":"Frequently asked questions"}}},{"node":{"id":"0d6f8da4-7d4f-5228-a832-7ff574e63570","frontmatter":{"path":"/frontmatter-guide","title":"Frontmatter-guide"}}},{"node":{"id":"249c8a64-20fb-510c-8b3e-f5b3b72fe028","frontmatter":{"path":"/grading/","title":"Grading"}}},{"node":{"id":"5b5e6522-0d23-577b-9ba9-4ecd1c1c5d24","frontmatter":{"path":"/","title":"About this course"}}},{"node":{"id":"bcdb2bdc-e92c-5c75-8dd1-13f57e1129da","frontmatter":{"path":"/glossary","title":"Glossary"}}},{"node":{"id":"d54c2afb-004d-500e-9d19-aa3d154a39a5","frontmatter":{"path":"/part-0/","title":"Part 0: See what an agent can build"}}},{"node":{"id":"fff8e65b-0281-55fe-a21a-9e7e872ddc33","frontmatter":{"path":"/part-1/1-getting-started/","title":"Getting started"}}},{"node":{"id":"9fa0a7fc-c673-56ad-9dc8-bf6aeb6d3b05","frontmatter":{"path":"/part-1/2-information-from-the-user/","title":"Information from the user"}}},{"node":{"id":"c2286d4a-4cf7-5fc4-ab0f-f7cd2acf49df","frontmatter":{"path":"/part-1/3-more-about-variables/","title":"More about variables"}}},{"node":{"id":"3ebadf79-0e16-5ed7-bcb2-bec3731cffc2","frontmatter":{"path":"/part-1/4-arithmetic-operations/","title":"Arithmetic operations"}}},{"node":{"id":"c77004ff-fe4d-5533-b77f-c861fa42430d","frontmatter":{"path":"/part-1/5-conditional-statements/","title":"Conditional statements"}}},{"node":{"id":"50ee91dc-7fe8-508d-940f-577449a335f4","frontmatter":{"path":"/part-1/","title":"Part 1"}}},{"node":{"id":"d963c6b4-b2d4-59ff-b310-265b28e2c8d2","frontmatter":{"path":"/part-2/1-programming-terminology/","title":"Programming terminology"}}},{"node":{"id":"075f597d-a49f-59c1-ad1a-d68ba19aa0f4","frontmatter":{"path":"/part-2/2-else-elif/","title":"More conditionals"}}},{"node":{"id":"26776fa9-27c7-5554-990b-09d6661a07a8","frontmatter":{"path":"/part-2/3-combining-conditions/","title":"Combining conditions"}}},{"node":{"id":"1da4e234-e81f-5494-b2d4-96e30cbb4464","frontmatter":{"path":"/part-2/4-simple-loops/","title":"Simple loops"}}},{"node":{"id":"db78d781-f405-5904-91bf-e96f7f04353f","frontmatter":{"path":"/part-2/","title":"Part 2"}}},{"node":{"id":"10808e92-b68f-58c3-b767-cf5e9c0efeb7","frontmatter":{"path":"/part-3/1-loops-with-conditions/","title":"Loops with conditions"}}},{"node":{"id":"bceed420-a129-52a7-81ed-dafd67ad38df","frontmatter":{"path":"/part-3/2-working-with-strings/","title":"Working with strings"}}},{"node":{"id":"e2dfd9d3-07fa-59a0-b748-370355606256","frontmatter":{"path":"/part-3/3-more-loops/","title":"More loops"}}},{"node":{"id":"5b546d28-ac00-564f-9d75-cc09c4382b64","frontmatter":{"path":"/part-3/4-defining-functions/","title":"Defining functions"}}},{"node":{"id":"12122cff-c09e-53f1-8b76-8517f845e08c","frontmatter":{"path":"/part-3/","title":"Part 3"}}},{"node":{"id":"8fbb5845-152f-5be7-ac92-fa280093b2a2","frontmatter":{"path":"/part-4/1-vscode/","title":"The Visual Studio Code editor, Python interpreter and built-in debugging tool"}}},{"node":{"id":"b8d090a9-7e49-5ab7-a5a1-d126179dd9a0","frontmatter":{"path":"/part-4/2-more-functions/","title":"More functions"}}},{"node":{"id":"271308ca-b267-5c86-8051-8a1a3480cfcb","frontmatter":{"path":"/part-4/3-lists/","title":"Lists"}}},{"node":{"id":"e5a8fd16-890d-509e-9666-9c622648c18a","frontmatter":{"path":"/part-4/4-definite-iteration/","title":"Definite iteration"}}},{"node":{"id":"ca4caea3-eaad-5cc3-a1fa-83b9bb90a25e","frontmatter":{"path":"/part-4/5-strings-and-lists/","title":"More strings and lists"}}},{"node":{"id":"adeebce2-76be-5528-90a1-6feaa066fcd8","frontmatter":{"path":"/part-4/6-data-structures/","title":"Data structures and data shapes"}}},{"node":{"id":"07e761ca-39a1-50a1-97d8-9149e4737e89","frontmatter":{"path":"/part-4/","title":"Part 4"}}},{"node":{"id":"80fe5d23-31eb-5ff6-b339-99e173e9279f","frontmatter":{"path":"/part-5/0-working-with-ai/","title":"Working with AI from Part 5 onward"}}},{"node":{"id":"111a6af4-b8a7-5d71-8966-f750f6c161a6","frontmatter":{"path":"/part-5/1-tables-as-nested-lists/","title":"Tables as nested lists"}}},{"node":{"id":"7e2cf429-03ea-5e8c-86ce-62c71c689a75","frontmatter":{"path":"/part-5/2-references/","title":"References and transformed results"}}},{"node":{"id":"3901dc36-3116-5444-9120-6b38454e245f","frontmatter":{"path":"/part-5/3-grouping-and-structured-records/","title":"Grouping and structured records"}}},{"node":{"id":"076a1db2-8d62-5725-ad72-c43ea25897c6","frontmatter":{"path":"/part-5/4-small-database/","title":"Building a small database"}}},{"node":{"id":"cf823ebf-3d67-53ca-8fcf-9b53fee10c1d","frontmatter":{"path":"/part-5/5-in-memory-analysis-pipeline/","title":"An in-memory analysis pipeline"}}},{"node":{"id":"92cbc670-d401-53b8-949f-9725b81a5e4a","frontmatter":{"path":"/part-5/","title":"Part 5"}}},{"node":{"id":"52ecf5a7-9266-58c8-bac5-407b1a29a9dd","frontmatter":{"path":"/part-6/2-writing-files/","title":"Writing and exporting files"}}},{"node":{"id":"a296984d-7503-558e-86da-ddb801f8a59b","frontmatter":{"path":"/part-6/3-scope-of-variables/","title":"Make data flow explicit"}}},{"node":{"id":"98bdbc42-0ebc-5f9c-b767-b987266a0040","frontmatter":{"path":"/part-6/4-modules/","title":"Use existing libraries"}}},{"node":{"id":"0ca5c730-bd8e-5099-9768-8fcb8eb0b7b4","frontmatter":{"path":"/part-6/1-reading-files/","title":"Reading files"}}},{"node":{"id":"11759056-5a15-5820-a693-30f4df62f118","frontmatter":{"path":"/part-6/","title":"Part 6"}}},{"node":{"id":"6e748518-2c76-5428-8779-0f0455b17a0d","frontmatter":{"path":"/part-7/0-dataframes-and-views/","title":"Working with pandas dataframes"}}},{"node":{"id":"047a5114-7ebb-5a5c-8ed0-eadf81564546","frontmatter":{"path":"/part-7/1-times-and-dates/","title":"Times and dates"}}},{"node":{"id":"7cc9d1d2-fb68-52da-b5ec-034bf9f4585f","frontmatter":{"path":"/part-7/3-more-features/","title":"More Python features"}}},{"node":{"id":"5055044d-7938-5e5d-be5c-dfb0ff2c81b4","frontmatter":{"path":"/part-7/2-data-processing/","title":"Data processing"}}},{"node":{"id":"cea3f325-56bb-5a93-885e-275e465c1ced","frontmatter":{"path":"/part-7/","title":"Part 7"}}},{"node":{"id":"3abea309-839b-562f-bb00-3be40d963200","frontmatter":{"path":"/part-8/","title":"Part 8"}}}]}},"pageContext":{}}, "staticQueryHashes": ["1357757885","994120085"]}