<?xml version="1.0" encoding="UTF-8"?>
<?xml-model href="../../common/schema/DHQauthor-TEI.rng"    type="application/xml" schematypens="http://relaxng.org/ns/structure/1.0" ?>
<?xml-model href="../../common/schema/DHQauthor-TEI.isosch" type="application/xml" schematypens="http://purl.oclc.org/dsdl/schematron"?>
<?xml-model href="../../common/schema/dhqTEI-ready.sch"     type="application/xml" schematypens="http://purl.oclc.org/dsdl/schematron"?>
<TEI xmlns=      "http://www.tei-c.org/ns/1.0"
     xmlns:cc=   "http://web.resource.org/cc/"
     xmlns:dhq=  "http://www.digitalhumanities.org/ns/dhq"
     xmlns:html= "http://www.w3.org/1999/xhtml"
     xmlns:mml=  "http://www.w3.org/1998/Math/MathML"
     xmlns:rdf=  "http://www.w3.org/1999/02/22-rdf-syntax-ns#">
   <teiHeader>
      <fileDesc>
         <titleStmt><!--Author should supply the title and personal information-->
            <title type="article" xml:lang="en">Entity Extraction in Premodern German Administrative Documents: Accuracy and Cost in the Age of Generative AI</title>
            <!--Add a <title> with appropriate @xml:lang for articles in languages other than English-->
            <dhq:authorInfo><!--Include a separate <dhq:authorInfo> element for each author-->
               <dhq:author_name>Ismail<dhq:family> Prada Ziegler</dhq:family>
               </dhq:author_name>
               <idno type="ORCID">https://orcid.org/0000-0003-4229-8688</idno>
               <dhq:affiliation>University of Bern</dhq:affiliation>
               <email>ismail.prada@unibe.ch</email>
               <dhq:bio>
                  <p/>
               </dhq:bio>
            </dhq:authorInfo>
         </titleStmt>
         <publicationStmt>
            <publisher>Alliance of Digital Humanities Organizations</publisher>
            <publisher>Association for Computers and the Humanities</publisher>
            <!--This information will be completed at publication-->
            <idno type="DHQarticle-id">000877</idno>
            <idno type="volume">020</idno>
            <idno type="issue">4</idno>
            <date><!--include @when with ISO date and also content in the form 23 February 2024--></date>
            <idno type="DOI">pending</idno>
            <dhq:articleType>article</dhq:articleType>
            <availability status="CC-BY-ND"><!--If using a different license from the default, choose one of the following:
                  CC-BY-ND (DHQ default):        
                  CC-BY:    
                  CC0:  -->
               <cc:License rdf:about="http://creativecommons.org/licenses/by-nd/2.5/"/>
            </availability>
         </publicationStmt>
         <sourceDesc>
            <p>This is the source</p>
         </sourceDesc>
      </fileDesc>
      <encodingDesc>
         <classDecl>
            <taxonomy xml:id="dhq_keywords">
               <bibl>DHQ classification scheme; full list available at <ref target="https://dhq.digitalhumanities.org/taxonomy.xml">https://dhq.digitalhumanities.org/taxonomy.xml</ref>
               </bibl>
            </taxonomy>
            <taxonomy xml:id="authorial_keywords">
               <bibl>Keywords supplied by author; no controlled vocabulary</bibl>
            </taxonomy>
            <taxonomy xml:id="project_keywords">
               <bibl>DHQ project registry; full list available at <ref target="https://dhq.digitalhumanities.org/projects.xml">https://dhq.digitalhumanities.org/projects.xml</ref>
               </bibl>
            </taxonomy>
         </classDecl>
      </encodingDesc>
      <profileDesc>
         <langUsage>
            <language ident="en" extent="original"/>
            <!--add <language> with appropriate @ident for any additional languages-->
         </langUsage>
         <textClass>
            <keywords scheme="#dhq_keywords"><!--Authors may suggest one or more keywords from the DHQ keyword list, visible at https://dhq.digitalhumanities.org/taxonomy.xml; these may be supplemented or modified by DHQ editors--><!--Enter keywords below preceeded by a "#". Create a new term element for each-->
               <term corresp=""/>
            </keywords>
            <keywords scheme="#authorial_keywords"><!--Authors may include one or more keywords (in <term> elements) of their choice-->
               <term/>
            </keywords>
            <keywords scheme="#project_keywords">
               <list type="simple">
                  <item/>
               </list>
            </keywords>
         </textClass>
      </profileDesc>
      <revisionDesc><!-- Replace both "NNNNNN"s in the @target of ther <ref> below with the appropriate DHQarticle-id value. -->
         <change>The version history for this file can be found on <ref type="gitHist" target="https://github.com/Digital-Humanities-Quarterly/dhq-journal/commits/main/articles/000877/000877.xml">GitHub</ref>.</change>
      </revisionDesc>
   </teiHeader>
   <text xml:lang="en" type="default">
      <front>
         <dhq:abstract>
            <p>Named Entity Recognition in historical texts faces unique challenges, including non-standardized language, transcription noise, and limited annotated material. We evaluate two approaches on two tasks on the Historical Land Records of Basel, a dataset of 829 German-language documents from the 14th to 18th century. Task 1 involves classic NER, identifying persons, locations, and organizations, while Task 2 addresses nested entity annotations, additionally requiring the identification of grammatical structures such as heads, appositions, and attributes. We compare generative LLM-based tagging (GPT-4o) with domain-specific fine-tuned sequence taggers using Flair contextual character embeddings, examining the impact of variable training data. GPT outperforms Flair in few-shot scenarios, particularly for low-frequency tags, while Flair achieves superior performance with larger training sets and longer spans. We find that, no matter which system, a substantial increase in accuracy can be gained by increasing the amount of training data. We also analyze annotation cost and speed, showing that manually curated datasets can be generated efficiently and support well-performing models. Our findings demonstrate that LLMs provide strong initial results, but lighter traditional systems remain competitive with sufficient training data.</p>
         </dhq:abstract>
         <dhq:teaser><!--Include a brief teaser, no more than a phrase or a single sentence-->
            <p/>
         </dhq:teaser>
      </front>
      <body>
         <div>
            <head>TEST
               Introduction</head>
            <p>Named Entity Recognition (NER) is a field of Natural Language Processing concerned with extracting references to entities - typically persons, places, and organizations, but sometimes extended to include works of art, brands, and other categories - from text. In historical editions, the annotation and linking of these entities was commonplace long before the advent of the internet. When entities are properly annotated and identified in a database, the resulting information can be used to build large-scale registries through which historians, sociologists, and other researchers, as well as the interested public can quickly find materials of interest. Understandably, an increasing number of historical projects are seeking to enrich their digitized datasets through automated methods. </p>
            
            <p>Historical texts, however, present a number of challenges for building well-performing NER models. In addition to the difficulties posed by high spelling variation and noisy transcriptions, the persistent lack of available training data has long hindered research in historical NLP <ptr target="#ehrmann2023"/>. The extent of this data scarcity is evident in the survey on premodern NER by <ptr target="#ehrmann2023"/>: among the datasets listed, only one German-language resource refers to documents produced before the 18th century. Even within that dataset, <title rend="italic">Deutsches ROman Corpus</title>, only two out of ninety novel fragments date from before 1700, and nine from before 1800 <ptr target="#krug2017"/>. Overall, just one dataset in the survey includes documents from medieval and early modern periods: a multilingual collection of annotated medieval charters <ptr target="#aguilar2022"/>.</p>
           
            <p>Unsurprisingly, researchers have sought solutions that require little or no training data. One such method was to rely on rule-based systems in combination with gazetteers - for example, lists of common person names <ptr target="#grover2008"/>. For historical texts that are linguistically close to their modern counterparts, a valid strategy is to apply a model trained on modern language and fine-tune it using a small amount of historical data <ptr target="#blouin2021"/>. </p>
            
            <p>More recently, with the rise of generative large language models (LLMs) such as GPT, zero- and few-shot approaches have become increasingly prevalent <ptr target="#gonzález2023"/>; <ptr target="#hiltman2025"/>.
               
               <note> Zero shot approaches rely only on pre-trained knowledge of a model, while few-shot approaches include a few examples for the model in the prompt.</note>
               
               Although most studies indicate that traditional methods, such as recurrent neural networks and BERT architectures, continue to outperform generative LLMs on the majority of information extraction tasks (<ptr target="#gonzález2023"/>; <ptr target="#wang2025"/>; <ptr target="#keraghel2024"/>), the ease of use and minimal preparation required to obtain acceptable results can tempt scholars in the (digital) humanities to rely exclusively on generative LLMs. Consequently, results produced by these models are often accepted as sufficiently accurate, and in some cases, no systematic evaluation of the automated output is conducted. </p>
            
            <p><hi rend="bold">This paper examines what we risk losing when depending solely on zero- and few-shot systems</hi>. We do so by presenting a case study that explores performance differences depending on the amount of available training material. Specifically, we evaluate two Named Entity Recognition strategies, one based on GPT-4o (<ptr target="#openai2023"/>) and the other on the Flair framework (<ptr target="#akbik2019"/>), using a historical dataset of German-language administrative documents. We then reflect on these methods with respect to both accuracy and financial cost.</p>
           
            <p>We evaluate the accuracy of the models in two distinct settings: First, we consider a standard NER task in which only proper names must be identified. Second, we address a nested entity recognition task employing an expanded tagset designed for more complex historical inquiries. Because GPT-4o possesses pre-trained knowledge about named entity recognition, our goal here is to assess how effectively the model can follow specific annotation guidelines in such complex contexts. <hi rend="bold">The case study further illustrates how the amount of available training data influences the performance of both approaches</hi>.</p>
         </div>
         <div>
            <head>Dataset</head>
            <p>The foundational dataset for this case study is the <title rend="italic">Historical Land Records of Basel Ground Truth</title> corpus (HLR-GT) <ptr target="#prada2025"/>. The corpus comprises 829 digitized and annotated archival index cards containing excerpts from administrative documents dating from the fourteenth to the eighteenth century. It provides an ideal scenario for Natural Language Processing on historical material, as it features all the characteristic challenges: noise introduced by handwritten text recognition (HTR), non-standardized language, scarcity of annotated material, and the absence of a suitable baseline model.</p>
            
            <!-- Figure 1 -->
            <figure>
               <head>One archival index card on which the HLR-GT is based. Only the main text is part of the corpus; header, marginalia, and footnotes were removed. Image Source: State Archive of Basel-Stadt HGB 1 1/34, p. 4.</head>
               <graphic url="resources/images/figure01.jpg"></graphic>
               <figDesc></figDesc>
            </figure>
            
            <p>From a scholarly perspective, the corpus offers a rich foundation of information well-suited for quantitative analysis. Each index card contains an excerpt or copy of an original document written in Early New High German. Most entries record a single transaction, trade, or list of dues payments, though less frequent document types, such as court cases or testaments, are also represented. While capitalization and punctuation are generally present, both are inconsistent; although this information can assist the models in learning and predicting annotations, it may also occasionally introduce noise. Documents from the same source (e.g., court books) often exhibit strong structural similarity. For instance, a property purchase frequently follows the formula: <quote rend="inline">&lt;Actor A&gt; sells &lt;Actor B&gt; &lt;Property&gt; for &lt;Money&gt;.</quote> Systems that learn to recognize such formulaic structures tend to perform better. However, they are also at risk of overfitting and may fail to generalize to documents that deviate from these patterns. These observations were previously noted in <ptr target="#prada2024"/>.</p>
            
            <!-- Figure 2 -->
               <figure>
                  <head>A BeNASch-annotated person reference in XML-encoding.</head>
                  <graphic url="resources/images/figure02.png"></graphic>
                  <figDesc></figDesc>
               </figure>
            
            <p>The HLR is annotated according to a variant of the BeNASch standard,
               <note> <ref target="https://dhbern.github.io/BeNASch/">https://dhbern.github.io/BeNASch/</ref> (last accessed 08.10.2025).</note>
               
               which employs a deeply nested entity annotation scheme. An example of this structure is shown in figure 2. In the excerpt, three separate entities are mentioned (<hi rend="italic">reference</hi>): <title rend="italic">Margareth Weibert</title>, <title rend="italic">Hanns Heinrich Wentzen</title>, and <title rend="italic">Greater Basel</title> (the part of the city west of the Rhine). Through nesting, additional layers of information are encoded. Heads, appositions, and attributes (the latter not present in this example) provide further details about each entity, including names (<hi rend="italic">nam</hi>), occupations (<hi rend="italic">occ</hi>), and family relationships (<hi rend="italic">fam</hi>). In subsequent processing, this nested information can be leveraged to extract relationships. For instance, <hi rend="italic">Margareth</hi> is related to <title rend="italic">Hanns</title> as <title rend="italic">Wittib</title> (widow), and <title rend="italic">Hanns</title> is employed by <title rend="italic">Greater Basel</title> as <title rend="italic">Stattgerichts Besitzer</title> (member of the court).</p>
            
            <p>For Task 1, representing a conventional named entity recognition setup, all annotations were removed except head elements with the <hi rend="italic">nam</hi> class. For Task 2, the full tagset was retained. The dataset comprises a total of 54,925 tokens across 829 documents. Document length varies considerably, with an average of 66.26 tokens and a standard deviation of 34.33 tokens. The shortest document contains 7 tokens, while the longest extends to 237 tokens.</p>
         </div>
         <div>
            <head>Approaches</head>
            <p>In this study, we compare two approaches to named entity recognition on historical texts: LLM-based tagging and specifically trained supervised taggers. For the latter, we developed and applied Flair models in three distinct configurations. In parallel, we implemented a commercially available LLM using a strategy that has proven successful in prior work.</p>
            
            <p>Flair leverages contextual character embeddings <ptr target="#akbik2018"/>. Unlike most other embedding strategies that rely on subword-based tokenization such as WordPiece that split words into common subword tokens (e.g. <quote rend="inline">pretrained</quote> becomes <quote rend="inline">pre,</quote> <quote rend="inline">train,</quote> <quote rend="inline">ed</quote>) and encode those, this strategy bases the encoding of tokens on their individual characters, as well as the context that comes before and after the token. This results in a smaller drop in performance when confronted with noise and variable spelling and, has demonstrated robust performance for premodern German <ptr target="#schweter2019"/>; <ptr target="#hodel2023"/>. In addition, because these embeddings, as well as the task-specific models, are trained using recurrent neural network architectures (Bi-LSTM), they require less training material than would be required to train or finetune a LLM like BERT <ptr target="#schweter2020"/>.
            </p>
            <div>
            <head>Flair: Base</head>
            <p>This approach employs the pre-trained Flair model <hi rend="italic">ner-german</hi>
               <note>
                  <ref target="https://huggingface.co/flair/ner-german">https://huggingface.co/flair/ner-german</ref> (last accessed: 09.10.2025)</note> <ptr target="#akbik2018"/>. This model was trained on the CoNLL-03 revised dataset (<ptr target="#tjong2003"/>), consisting primarily of modern, annotated newspaper articles. Using this model out of the box is the quickest approach, but its lack of knowledge about the target domain is expected to limit performance. It is important to note that the annotation guidelines for CoNLL-03 and the reduced HLR-GT dataset are not fully aligned. Hence some annotations that would be considered correct under CoNLL-03 standards may not be recognized as correct in our evaluation.</p>
            </div>
            <div>
            <head>Flair: Fine-tuned</head>
            <p>In this approach, the pre-trained <hi rend="italic">ner-german</hi> model is fine-tuned on the HLR-GT dataset. Fine-tuning is a suitable strategy when only limited raw data is available and training a language model from scratch is impractical. The objective is to enrich the base model with domain-specific knowledge as well as familiarity with the desired annotation guidelines. Fine-tuning was performed using the available training data for each scenario, with a learning rate of 0.00005 over 50 epochs. The number of epochs was determined through evaluation on the validation set during initial testing.</p>
            </div><!-- fine tuned -->
            <div>
            <head>Flair: Customized</head>
            <p>In this approach, we fine-tuned contextual character embeddings, the <hi rend="italic">de-forward</hi> and <hi rend="italic">de-backward</hi> models provided by Flair, using all available text from the historical land records<note> Not published at this date.</note> (including documents which are not part of the annotated dataset, totaling approximately 9 million tokens). Using these fine-tuned embeddings, we then trained custom SequenceTagger models in Flair with default parameters.<note> Sequence Tagger models in Flair are simple Bi-LSTM+CRF architectures.</note>
            </p>
            </div><!-- flair customized -->
            <div>
            <head>Other “Traditional” Embeddings</head>
            <p>In preliminary experiments, we investigated the performance of BERT (<ptr target="#devlin2019"/>), fastText (<ptr target="#bojanowski2017"/>), and word2vec embeddings (<ptr target="#mikolov2013"/>) in combination with the Flair sequence tagger, both with and without stacking alongside contextual character embeddings. When used without stacking, performance was significantly worse, while stacking did not yield any significant improvement. Therefore, these embeddings were not included in the experiments reported in this case study.</p>
            </div>
            <div>
            <head>Generative AI (GPT-4o)</head>
            <p>Several strategies have been proposed for prompting generative LLMs to perform entity extraction. In this study, we follow the recent approach by <ptr target="#hiltman2025"/> and conduct our experiments with the framework NER4all.
               
               <note> Our version of the code, including the additions mentioned in this paper is available at <ref target="https://github.com/raykyn/ner4all-ext-clean">https://github.com/raykyn/ner4all-ext-clean</ref>. The original NER4all code has since been refactored and been published under the name annotations4all under   <ref target="https://scm.cms.hu-berlin.de/annotations4all/annotations4all">https://scm.cms.hu-berlin.de/annotations4all/annotations4all</ref>.</note>
               
               All experiments were executed using <hi rend="italic">gpt-4o-2024-08-06.</hi> In the following sections, we detail the changes we made to the default NER4all configuration.</p>
               
               <div>
            <head>Detailed Context Prompt</head>
                  <p>Following the recommendations of <ptr target="#hiltman2025"/>, we augmented the prompt with detailed contextual information. We selected a German full prompt, which demonstrated slightly better performance on the validation set. Further task-specific fine-tuning and adaptation of the prompt are described in the relevant sections. All prompt fine-tuning was conducted using a validation set comprising 83 documents. Complete prompts are provided in the appendix.</p>
               </div>
               <div>
                  <head>Similarity-based Examples</head>
                  <p>As part of the prompt, the system is provided with a set of examples, which is particularly important for learning non-standard annotation guidelines. In <ptr target="#hiltman2025"/>, examples were manually selected to ensure coverage of all tags in the tagset.</p>
           
                  <p>When we simulate scenarios with limited training data in the following experiments, if the available training data is smaller than the number of shots, the examples are chosen randomly. Curated examples can perform better in such cases, as random selection may omit some entity classes or include only generic examples.</p>
                  
            <p>Given that the number of available examples often exceeds the prompt’s context length, we implemented a strategy proposed by <ptr target="#wang2025"/>, who found that selecting contextually appropriate examples rather than random ones can substantially improve model output. The strategy aims to find examples that are similar to the given input. We use the <title rend="italic">DocumentPoolEmbeddings</title> provided by the Flair framework to combine the language models trained for the <hi rend="italic">flair-custom</hi> model to generate embeddings for all samples in our training data. During prompt construction, the input sample is embedded using the same method, and cosine distances to all training samples are calculated. The examples with the smallest distances, i.e., the most similar samples, are then selected to be presented to the model.</p>
            
                  <table><!-- Table 1 -->
               <head>An example of how the similarity-based retrieval works to get annotation examples of different house references. Tags are omitted in favor of readability; the prompt will include each example with and without tags.</head>
               <row role="data">
                  <cell>Input</cell>
                  <cell>seinem Haus zur Büttenen genannt bey sant Urbans Brung nen gelegen</cell>
               </row>
               <row role="data">
                  <cell>1</cell>
                  <cell>seinem Eckhaus hinder sant Elsbethen ob des Spittal Kornhaus zum Widerlin genannt gelegen</cell>
               </row>
               <row role="data">
                  <cell>2</cell>
                  <cell>seinem hauss zum Fürst genant an der Gerbergassen , gelegen</cell>
               </row>
               <row role="data">
                  <cell>3</cell>
                  <cell>seinem Haus gegen sant Elsbethen überhin gelegen</cell>
               </row>
            </table>
               </div>
               <div>
                  <head>Increased Shots</head>
                  <p><ptr target="#hiltman2025"/> report that a maximum of 32 examples (<quote rend="inline">shots</quote>) is typically optimal due to the context window limitations of the models. In our experiments, we evaluated different shot counts on the validation set and selected the configuration that yielded the best performance: 128 shots for Task 1 and 64 shots for Task 2. Although the performance differences between 32, 64, and 128 shots were generally small, they were statistically significant. In Task 2, for instance, reducing the number of shots from 64 to 32 resulted in an approximate 2 percentage point decrease in performance.</p>
               </div>
            </div>
         </div>
         <div>
            <head>Accuracy</head>
            <div>
               <head>Task 1: Named Entity Extraction</head>
               <div>
                  <head>Task Description</head>
                  <p>In this task, the models are required to identify conventional named entity spans, typically corresponding to proper nouns. The target categories are persons (PER), organizations (ORG), and locations (LOC). Geopolitical entities, which are tagged separately in the HLR-GT dataset, were mapped to these categories depending on context (e.g., <quote rend="inline">I live in Basel</quote> ⇒ LOC, <quote rend="inline">I am a citizen of Basel</quote> ⇒ ORG).</p>
                 
                  <p>The HLR-GT contains 2,583 PER tags, 2,079 LOC tags, and 518 ORG tags.</p>
                  
                  <p>To assess the effect of available training data, models were trained or prompted using subsets of 4, 8, 16, 32, 64, 128, 256, 512, and 662/745 samples.
                     
                     <note> The maximum available training sample count varies slightly, because the sample count for Flair always includes the 10% of samples used for validation, while the sample count for GPT is exclusively using the training data at this point.</note>
                     
                     Thresholds of 4 and 8 were omitted for Flair models, as performance at these levels was entirely unusable. Each experimental configuration was executed three times to improve reliability of the results.</p>
               </div><!-- task description -->
               <div>
                  <head>Generative AI Setup</head>
                  <p>Validation set experiments indicated that a 128-shot configuration was optimal for this task. Based on a review of NER4all’s performance on the validation set, we introduced additional instructions to address recurring errors in the model output. Specifically, the guidelines were extended as follows: 1) Annotate only proper names. 2) Assign geopolitical entities to the appropriate category based on context. 3) For organizations and locations, annotate the grammatical head. 4) Do not annotate names of saints when they occur as part of other entity names.</p>
               </div>
               
               <div>
                  <head>Results</head>
                  <p>The results are plotted in figure 3 and listed as a table in appendix D. As expected, GPT-4o dominates the few-shot scenarios, outperforming the best-performing Flair models by 5-10 percentage points. Consistent with prior findings (<ptr target="#hiltman2025"/>), the non-fine-tuned Flair model performs substantially worse than all other approaches, likely due to its inability to handle the historical language and lack of knowledge regarding the annotation guidelines. Among the Flair models, the custom model trained from scratch achieved the highest performance, surpassing even the fine-tuned Flair model. The fine-tuned model’s relatively lower performance appears to result from the absence of a corpus-specific language model. Regarding GPT-4o, there appears to be a critical threshold in the number of examples provided: performance increases sharply from 16 samples (68.76% F1) to 32 samples (74.51% F1), with further significant gains observed as the number of samples doubles up to 256. Beyond this point, performance begins to plateau, with a slight upward trend observed at 662 samples. For Flair-Custom, results are highly variable when training data is limited. This may be attributable to the relatively small validation set, comprising approximately 10% of the total documents (e.g., only 3 documents in the validation set for the 32-sample scenario). A small validation set may cause training to terminate before an optimal model state is reached; however, initial testing indicated that increasing the validation set proportion generally worsened average performance. At around 128 samples, Flair-Custom matches GPT-4o performance, while at 256 samples, it outperforms all other models. Performance appears to plateau after 512 samples.</p>
                  
                  <!-- Figure 3 -->
                  <figure>
                     <head>Performance comparison of chosen models on different training data settings.</head>
                     <graphic url="resources/images/figure03.png"></graphic>
                     <figDesc></figDesc>
                  </figure>
                  
               </div>
               <div>
                  <head>Error Analysis</head>
                  <p>To investigate the observed performance plateaus, we analyzed predictions classified as incorrect during automated evaluation. We found that 66% of Flair’s errors and 54% of GPT-4o’s errors could be attributed to one of the following categories:</p>
                  <list type="ordered">
                     <item><hi rend="italic">HTR-induced noise:</hi> Errors stemming from heavy transcription errors, where even human annotators could only roughly infer the correct annotation (Flair: 13%, GPT-4o: 8%).</item>
                     <item><hi rend="italic">Ground truth inaccuracies:</hi> Cases in which the model’s prediction was correct, but the ground truth was erroneous (Flair: 19%, GPT-4o: 13%).</item>
                     <item><hi rend="italic">Reasonable deviations:</hi> Predictions that differed from the ground truth but  were reasonable enough that they would likely not significantly affect downstream tasks, such as entity linking (Flair: 34%, GPT-4o: 33%). For example, the ground truth might mark “frowen zu Clingental” as the entity mention while the model only annotated “Clingental”, with both phrases referring to the same concept of the monastery of Klingental. These errors often mirror inconsistencies in the training data.</item>
                  </list>
                  <p>In addition, approximately 10% of errors in both models were simple ORG-–LOC confusions. While potentially more impactful for downstream tasks such as event recognition, these errors are of limited relevance for tasks like creating a corpus index. Additionally, GPT-4o occasionally annotated non-named entity references (e.g., <quote rend="inline">Brugg</quote> as a bridge), which accounted for 10% of its errors. Again, for most downstream tasks, this would not be particularly problematic behaviour. Overall, only 28% of GPT-4o’s errors and 24% of Flair’s errors could not be explained by one of the categories above. Given the already strong performance of both models, this analysis suggests that further improvements would require substantially larger amounts of training data.</p>
               </div>
            </div><!-- end task 1 -->
            <div>
               <head>Task 2: Nested Extraction</head>
               <div>
                  <head>Task Description</head>
                  <p>The HLR-GT dataset features a complex nested annotation scheme, in which entity references are further annotated to capture additional information about the entities they contain. Task 2 evaluates model performance on the full annotation set. Each tag consists of an element tag (e.g., <hi rend="italic">reference</hi>) and, in most cases, an associated class tag (e.g., <hi rend="italic">person</hi>). An overview of element tags is provided in table 2.</p>
                  <table><!-- table 2 -->
                     <head>Overview of the element tags the systems must detect in task 2.</head>
                     <row role="data">
                        <cell>Tag</cell>
                        <cell>Description</cell>
                        <cell>Example</cell>
                     </row>
                     <row role="data">
                        <cell>REF</cell>
                        <cell>Entity reference</cell>
                        <cell>Johann the tailor</cell>
                     </row>
                     <row role="data">
                        <cell>HEAD</cell>
                        <cell>Head element in reference</cell>
                        <cell>Johann the tailor</cell>
                     </row>
                     <row role="data">
                        <cell>APPO</cell>
                        <cell>Apposition in reference</cell>
                        <cell>Johann the tailor</cell>
                     </row>
                     <row role="data">
                        <cell>ATTR</cell>
                        <cell>Attribute in reference</cell>
                        <cell>the house in Basel</cell>
                     </row>
                     <row role="data">
                        <cell>DATE</cell>
                        <cell>Reference to a date</cell>
                        <cell>1573 / sunday after martini</cell>
                     </row>
                     <row role="data">
                        <cell>CURR</cell>
                        <cell>Monetary value or similar</cell>
                        <cell>5 lb / 2 satchels of corn </cell>
                     </row>
                     <row role="data">
                        <cell>EV</cell>
                        <cell>Event Trigger</cell>
                        <cell>Johann sells a house</cell>
                     </row>
                  </table>
                  
                  <p>For references, event triggers, and descriptive elements (HEAD, APPO, ATTR), a class tag must also be identified. The full combined element+class tagset comprises 80 tags and is provided in Appendix E, including tag frequencies. Further details on the tagset are available in the HLR-GT annotation documentation.
                     
                     <note> <ref target="https://doi.org/10.5281/zenodo.16919653">https://doi.org/10.5281/zenodo.16919653</ref> (last accessed: 09.10.2025)</note>
                  </p>
                  <p>As in task 1, we examine the impact of available training data by training or prompting models with varying numbers of examples. Each experimental configuration was executed three times to improve reliability. For this task, we restrict Flair experiments to the <hi rend="italic">flair-custom</hi> model.</p>
               </div>
               <div>
                  <head>Flair Setup</head>
                  <p>The Flair SequenceTagger architecture can be adapted to handle nested annotations using a bottom-up recursive approach <ptr target="#prada2024b"/>. In the first step, the entire text is used as input. If entity references are detected, the text inside the entity reference is re-annotated by the model. For elements that allow nested content, namely, entity references, appositions, and attributes, additional annotation passes are performed recursively until no further nested elements remain. The final output consists of all predicted tags collected throughout this process. Each input span is pre- and suffixed with a tag indicating its type, for example:</p>
                  
                  <p><quote rend="block">[B-REFERENCE-PER] <name>Johann Müller</name>, the tailor [E-REFERENCE-PER].</quote></p>
                  
                  <p>For model training, the dataset is prepared so that every element potentially containing nested annotations (documents, references, appositions, and attributes) is represented as a separate training sample. Only immediately nested tags are included as targets:</p>
                  
                  <p><quote rend="block">[B-REFERENCE-PER] &lt;HEAD&gt;Johann Müller&lt;/HEAD&gt; , &lt;APPO-OCC&gt;the tayilor&lt;/APPO-OCC&gt; [E-REFERENCE-PER].</quote></p>
                  
                  <p>In this example, the head inside <quote rend="inline">the tailor</quote> is not annotated because it is not directly connected to the reference; a separate sample handles this annotation.</p>
               </div>
               <div>
                  <head>Generative AI Setup</head>
                  <p>NER4all does not natively support nested annotations. To address this, we evaluated two methods of our own design: the single prompt method and the recursive prompting method. In both approaches, we found that 64 shots provided optimal performance.</p>
                  <div>
                     <head>Single Prompt Method</head>
                     <p>Generative AI models are capable of producing strings with nested annotations, for example: <quote rend="inline">&lt;&lt;PER &lt;&lt;HEAD Johann /HEAD&gt;&gt; &lt;&lt;APPO the tailor /APPO&gt;&gt; /PER&gt;&gt;.</quote> We adapted the prompts and examples accordingly and modified NER4all to decode nested annotations while retaining its ability to correct the output if it deviates from the input. The full prompt is provided in appendix C.</p>
                  </div>
                  <div>
                     <head>Recursive Prompting Method</head>
                     <p>This method mirrors the bottom-up recursive approach used in the Flair pipeline. At each annotation layer, a suitable prompt is selected, and examples are generated according to the layer. A major drawback of this approach is an increase in API calls by approximately a factor of 15 compared to the single prompt method. This estimate includes a safeguard that first checks whether the input is already present in the training data, reducing required calls by about 20%.</p>
                  </div>
               </div>
               <div>
                  <head>Results</head>
                  <p>The results for task 2 are presented in figure 4. Consistent with task 1, both GPT-based systems outperform Flair in few-shot scenarios. The recursive GPT-4o system significantly outperforms the single prompt approach. At 64 documents, Flair performs on par with the recursive GPT system (67.36% vs. 66.75% F1). With 128 documents, Flair marginally outperforms the recursive GPT system (71.45% vs. 70.20% F1). At 256 documents Flair achieves a clear advantage (76.45% vs. 73.52% F1), and this performance margin continues to grow with additional training data. GPT performances appear to plateau between 512 and 662 training samples, while <hi rend="italic">flair-custom</hi> continues to improve.</p>
                  
                  <p>Closer examination reveals that recursive GPT occasionally generates its own tag categories. Depending on the amount of training data, hallucinated tags accounted for 2–5% of total incorrect predictions, particularly introducing new event trigger classes. While advanced post-processing could potentially map these tags to existing categories and improve accuracy, we did not further explore this.</p>
                 
                 <!-- figure 4 -->
                  <figure>
                     <head>Performance comparison of chosen models on different training data settings.</head>
                     <graphic url="resources/images/figure04.png"></graphic>
                     <figDesc></figDesc>
                  </figure>
                 
                  <div>
                     <head>Tag Frequency Analysis</head>
                     <p>We investigated how tag frequency impacts model performance, which is particularly relevant given the highly unbalanced tagset of HLR-GT. Some tags, such as a role apposition (e.g., <quote rend="inline">Johann, the seller</quote>), appear only 12 times in the training corpus, whereas the most common tag - the proper name head - appears 4,027 times. </p>
                     
                     <p>Figure 5 compares the <hi rend="italic">flair-custom</hi> model with the recursive GPT system using 128 training samples. We find that the models are affected to different degrees by an unbalanced tagset. Recursive GPT generally outperforms Flair on low-frequency tags. For high-frequency tags, GPT also surpasses Flair in some cases: out of 25 tags with frequency greater than 100, GPT performs better on 9. Notable large margins are observed for appositions of class <hi rend="italic">family</hi> (e.g., “Anna, Johann’s wife”) and event triggers of the sale class.</p>
                     
                     <p>For most high-frequency annotations, Flair slightly outperforms recursive GPT; however, there are some tags that see an even larger advantage for Flair: <hi rend="italic">facility</hi> references (marking houses), as well as references to <hi rend="italic">organizations</hi>. The stronger performance on facility tags can be explained by the following tag length evaluation. We suspect that in the case of the <hi rend="italic">organization</hi> tag that GPT suffers due to a lack of background knowledge. For instance, the phrase <quote rend="inline">die Frauen an den Steinen</quote> (the women at the Stones) refers to a particular monastery and was thus annotated as an organization in the HLR-GT. If GPT does not encounter this phrase among the selected examples, it cannot know about this and may misclassify the phrase as a reference to a person (group). Flair is less affected because, in the case of the 128-sample scenario, it has seen all samples (minus 12 for validation), whereas GPT only receives the most similar examples, which might not contain the necessary entity.</p>
                     
                     <!-- figure 5 -->
                     <figure>
                        <head>Flair compared to the recursive GPT system with 128 documents as sample data. Tag occurrence was measured on the full corpus. Note that for low-frequency tags, there’s the possibility that this tag could not be seen by a system at all. For the line plot, tags were sorted into buckets and averages calculated for each bucket. Note that the x-axis is log-scaled.</head>
                        <graphic url="resources/images/figure05.png"></graphic>
                        <figDesc></figDesc>
                     </figure>
                     
                  </div>
                  <div>
                     <head>Tag Length Analysis</head>
                     <p>We observed that the recursive GPT system struggles more with annotating very long spans compared to Flair. Figure 6 visualizes these differences. For annotations up to approximately 160 characters, the performance gap is minimal, with Flair slightly outperforming GPT. However, for spans exceeding 160 characters, Flair clearly demonstrates superior performance. Long spans typically correspond to house descriptions, such as the following example:</p>
                     
                     <p><quote rend="block">jrem Hus &amp; Hofstatt genant Nüwenstein , an der gerwergassen zwüschen dem Hus Helfenstein ze einer , u . Hanns ful des scherers hus zer andern siten , zinset vormals voneig 1sh . zu s . Lienh . sodan 1 fl den frauen an den steinen , widerk . mit 20fl .</quote></p>
                     
                     <p><quote rend="block"><hi rend="italic">English Translation: </hi>their house &amp; homestead called Nüwenstein, at the Gerwergassen between the house Helfenstein on one, and Hanns Ful the barbers' house on the other side, pays dues already for hereditary reasons 1 sh. to St. Lienh. also 1 pound to the women at the Stones, redeemable with 20 pounds.</quote></p>
                     
                     <p>Incomplete annotation of such spans can result in the loss of critical information, for example, the dues obligations associated with the property. This may also cause follow-up errors, like a dues or location attribute cut short.</p>
                    
                     <!-- figure 6 -->
                     <figure>
                        <head>Flair compared to the recursive GPT system with 128 documents as sample data with respect to their performance depending on annotation length (in characters). For the line plot, tags were sorted into buckets and averages calculated for each bucket. The grey bars indicate how frequent annotations of that length appear in the corpus.</head>
                        <graphic url="resources/images/figure06.png"></graphic>
                        <figDesc></figDesc>
                     </figure>
                     
                  </div>
               </div>
               <div>
                  <head>Inspecting the Model Performance Qualitatively</head>
                  <p>To illustrate how increased training data improves model performance, we present an example document from HLR-GT. The German text reads:</p>
                  
                  <p><quote rend="block">Es verkauft Herr Hanns Ulrich Falckhner und gibt zu Kaufen Frawen Margareth Weibert , weiland H . Hanns Heinrich Wentzen Stattgerichts Besitzers in Mehrerem Basell , hinderlassener Wittib eine Behausung bey dem schwartzen Pfol , einseits neben Herrn Bonaventura von Braun , Pfarhl bey St . Peter , andseits neben H . Johann Curtzen dem Jüngern gelegen .</quote></p>
                  
                  <p>An approximate English translation is:</p>
                  
                  <p><quote rend="block">It is sold by <name>Mr. Hanns Ulrich Falckhner</name>, and he gives to purchase to Lady Margareth Weibert, the late <name>Mr. Hanns Heinrich Wentzen</name>, a member of the city court in Greater Basel, widow, a house, at the Schwartzen Pfol, on one side next to <name>Mr. Bonaventura von Braun</name>, pastor at St. Peter, on the other side next to <name>Mr. Johann Curtzen the Younger</name>, located.</quote></p>
                  
                  <p>We intentionally chose a document that is rather simple compared to others and where we observed better predictions with increased training data. This is, of course, not true for most documents, as the inherent randomness of machine learning algorithms may, in specific cases, cause worse models to result in better performance.</p>
                  
                  <p>Beyond entity identification, models must correctly connect entities and classify information about them. Relevant facts include: Wentzen’s role as a city court member, Weibert as his widow, the house location, and the occupation of the pastor von Braun. Additionally, the sale event must be annotated. In the HLR scheme, repeated triggers are annotated only on the first occurrence, so either <quote rend="inline">verkauft</quote> (sold) or <quote rend="inline">Kaufen</quote> (purchase) would be acceptable in a wider sense, but only one is tagged.</p>
                 
                  <p>We compare annotations across five scenarios with increasing training data: recursive GPT-4o with 32 and 64 samples, Flair with 128, 256, and 512 samples. For clarity, each model is referred to by its number of training samples, and differences in annotation are highlighted for each part of the document.</p> 
                  <list type="simple">
                     <item><hi rend="bold"><quote rend="inline">verkauft</quote> (sold)</hi></item>
                     <item>All models correctly identified the span “verkauft” as a trigger for the sale event.</item>
                  </list>
                  
                  <list type="simple">
                     <item><hi rend="bold">The seller</hi></item>
                     <item>The full span “Herr Hanns Ulrich Falckhner” was correctly identified by all models as a person reference, with the proper name annotated as a <hi rend="italic">head </hi>element of class <hi rend="italic">nam</hi>.</item>
                  </list>
                  
                <list>
                   <item><hi rend="bold"><quote rend="inline">gibt zu kaufen</quote> (gives to purchase)</hi></item>
                   <item>The double expression of the event posed challenges for most models. Models with 32 and 64 training examples tagged “gibt zu kaufen” as an additional <hi rend="italic">sales </hi>trigger, while the 128-example model incorrectly tagged “gibt” as a <hi rend="italic">dues</hi> trigger and “Kaufen” as a <hi rend="italic">sales </hi>trigger. The 256-example model annotated both spans incorrectly as <hi rend="italic">sales </hi>triggers, and only the 512-example model correctly avoided any unnecessary annotations.</item>
                </list>
                  
                  <p>Notably, the GPT models outperform most Flair models in this scenario, even when trained on fewer examples. This observation aligns with findings from previous sections, highlighting the influence of tag frequency on accurate prediction. Event triggers are among the least frequently occurring annotations in the dataset, which can lead to inconsistent predictions in models trained on limited data.</p>
                  
                  <list>
                     <item><hi rend="bold">The buyer</hi></item>
                     <item>This span represents the most complex annotation challenge. It consists of the long, heavily nested reference:</item>
                  </list>
               
                  <p><quote rend="block">Frawen Margareth Weibert, weiland H . Hanns Heinrich Wentzen Stattgerichts Besitzers in Mehrerem Basell , hinderlassener Wittib</quote></p>
                  
                  <p>All models correctly identified the head “Margareth Weibert”. However, the nested structure caused errors for all models except the 512-sample model. </p>
                  
                  <p>Model 256 was mostly correct, but annotated the widow apposition “Weiland H . Hanns […] hinderlassener Wittib” as a separate person reference. </p>
                  
                  <p>Model 128 made the same mistake, additionally splitting “Mehrerem Basell” into two references and incorrectly tagging “Frawen” as a <hi rend="italic">name</hi> head in addition to the correct head, likely misled by capitalization. Model 32 failed to identify the widow apposition, resulting in a missing relationship between Weibert and Wentzen. It also incorrectly attached “in Mehrerem Basell” directly as an attribute to Wentzen instead of the occupation apposition “Stattgerichts Besitzers in Mehrerem Basell”. </p>
                  
                  <p>Model 64 recognized that the widow phrase belongs to Margareth Weibert, but misassigned “hinderlassener Wittib” as a head, and nested “weiland H . Hanns Heinrich Wentzen” into an attribute of class <hi rend="italic">dead</hi> (which doesn’t make any sense). Finally it incorrectly added the occupation apposition “Stattgerichts Besitzers in Mehrerem Basell” to Margareth Weibert. </p>
                  <p>In summary, the 64-sample model produced highly misleading annotations, while the 32-, 128-, and 256-sample models were incorrect but less misleading. Only the 512-sample model predicted this span correctly without errors.</p>
                  
                  <list type="simple">
                     <item><hi rend="bold">The house</hi></item>
                     <item>This span presented fewer difficulties than anticipated. All models except the 32-example model correctly predicted the full reference:</item>
                  </list>
    
                  <p><quote rend="block">“eine Behausung bey dem schwartzen Pfol , einseits neben Herrn Bonaventura von Braun , Pfarhl bey St . Peter , andseits neben H . Johann Curtzen dem Jüngern gelegen”</quote> </p>
                  
                  <p>and correctly identified the head “Behausung”. Model 32 only included “bey dem schwartzen Pfol” as an attribute describing the house's location and annotated the remaining entity references separately. Both models 32 and 64 struggled with the occupation apposition of Bonaventura von Braun. Model 32 annotated “Pfarhl bey St . Peter” as a separate location reference, likely thrown off by the HTR error (“Pfarrhr” is likely the correct phrase). Model 64 correctly annotates the apposition but fails to predict “St . Peter” as an organization reference, instead adding it as part of the head. The models 128, 256, and 512 all correctly predict this part. </p>
                  <list>
                     <item><hi rend="bold">Summary</hi></item>
                     <item>This example demonstrates that increasing the amount of training data results in more consistent and contextually accurate annotations, even for relatively straightforward documents. While all models correctly identified core elements, such as the sales event and principal actors, the precision and correctness of nested structures improved substantially with larger training sets. The model trained on 512 samples produced the only fully correct annotation, accurately handling the complex buyer reference with multiple nested attributes and relationships.</item>
                  </list>
               </div>
            </div>
         </div>
         <div>
            <head>The Cost of Automated Entity Extraction</head>
            <p>Using GPT-4o for annotation, we observed an average cost of $0.03 per document for task 1, and $0.12 per document for task 2 when employing the recursive prompting method. For large-scale collections, total costs may reach several thousand or even tens of thousands of dollars. OpenAI provides a batch API option, which reduces costs by approximately 50%, though this has not yet been implemented in NER4all, and it would be very difficult to combine with any recursive approaches. Reducing the number of examples in the prompt is another way to lower expenses, however this compromises performance. Open-weight models offer a cost-free alternative if suitable infrastructure is available to a research team; however, our experiments with GPT-OSS (<ptr target="#openai2023"/>) showed significantly worse performance than GPT-4o. The commercial model outperformed the locally hosted model by more than 15 percentage points in F1-Score in all scenarios. Further experiments with Deepseek-r1-70b (<ptr target="#deepseek2025"/>) did not produce viable outputs, as the model struggled with the encoding of the tags in the output.
               
               <!-- open ai 2023 only citation -->
               
               <note> We tried using different encoding strategies, such as having it encode in XML-style, but were unable to find a viable solution.</note>
               
            </p>
            <p>To give a pointer on how much creation of training data costs, we timed the annotation progress of a PhD student, who worked with and annotated the dataset previously. The PhD student was able to annotate on average 94 documents (7854 tokens, 636 annotations) per hour when utilizing the simple tagset from task 1. Annotation speed for the complex tagset in task 2 was considerably lower with appr. 16 documents (1212 tokens, 526 annotations) per hour.
               
               <note> Annotation was done on a self-hosted instance of INCEpTION <ptr target="#klie2018"/>. INCEpTION also features the option to use recommendation modules, so if a certain amount of training data has been created, by either method - or a base model is available for the target dataset - a recommender can help to speed up annotation efforts considerably. Considering most annotations stem from high frequency tags, the annotators can focus solely on fixing difficult passages. Although note that recommenders can quickly give a sense of security and annotators may be more prone to overlook mistakes when using a recommender.</note>
               
            </p>
            <p>For the simple tagset, a single experienced scholar could feasibly annotate sufficient data within less than one workday to train a Flair model that outperforms GPT-4o (on this dataset). For the complex tagset, generating enough high-quality training data requires multiple days of work. Additionally, best practices involve multiple annotators for the same documents and a curation process to resolve conflicts, further increasing time and personnel costs. Regardless of the system used, the creation and publication of annotated datasets constitute a valuable contribution to the digital humanities, particularly when adhering to FAIR principles, especially reusability. As demonstrated, even if insufficient data exists to train a fully competitive Flair model, GPT-4o benefits substantially from a larger pool of annotated examples.</p>
            
            <p>Another key factor to consider is the speed of the annotation systems. Flair trains very quickly, requiring approximately 10 minutes for task 1 and 60–90 minutes for task 2, and deploys even faster: the recursive annotation of the 84-document test set was completed in under one minute on an RTX 3090.
               
               <note> Inference is also possible with no video card available, but considerably slower with appr. 1 document per minute in recursive mode.</note>
               
               In contrast, GPT-4o predicted the same test set in approximately 80 minutes, with similarity-based example selection having negligible impact on processing time when run on an RTX 4070 SUPER. Depending on the size of the document collection, this difference may be a relevant criterion when selecting a system. This limitation is also why we did not conduct this study with GPT-5; a single pass over the test set in the recursive scenario could exceed 10 hours, making it impractical for use in our project despite its potential performance advantages over GPT-4o.</p>
         </div>
         <div>
            <head>Limitations and Future Work</head>
            <p>While this research allows for some more general conclusions, it is ultimately a single case study based on premodern German administrative records with a very specific language and structure. The specific performance thresholds and optimal data sizes observed here will naturally vary across different projects. A multitude of factors dictate the baseline difficulty of any extraction task, including the general complexity of the language, the specific phrasing used in the documents, the depth of the chosen annotation scheme, and the severity of the noise introduced by HTR errors and inherent spelling variation. We want to underline the importance of further similar case studies and evaluations of historical NER to help future researchers find their optimal strategies for automated information retrieval.</p>
            
            <p>Our investigation into the effects of tag frequency and annotation length revealed distinct, complementary strengths and weaknesses for both approaches. Future research should focus on combining these methodologies to create synergies and ultimately build better-performing systems. For example, considering the observed weakness of GPT-4o in correctly classifying entities it has not explicitly seen in its prompt examples, future work could explore entity-embedding-based example selection following <ptr target="#wang2025"/> to improve the coverage of unseen entities.</p>
            
            <p>Finally, our generative AI experiments focused on a commercial, closed-weight model. Although our preliminary tests with smaller open-weight models did not yield competitive results, the rapidly evolving landscape of open-weight LLMs warrants a much deeper investigation. Especially considering our experiments with Deepseek failed due to the encoding strategy employed by NER4all, evaluating stronger local models with different strategies remains a priority for further research.</p>
            
         </div>
         <div>
            <head>Conclusion</head>
            <p>
               This study addresses a practical dilemma for projects in history (and the humanities more generally): should researchers rely on modern generative AI, which provides immediate results, or invest time in manually curating data to train specialized, traditional, controllable systems? Our findings demonstrate a trade-off. Generative AI acts as a capable assistant out-of-the-box, providing a strong starting point when historical training data is virtually nonexistent. However, it may struggle to adapt to complex, project-specific rules, and the noise and language inherent to historical documents. Both, traditional models, represented in these experiments by Flair, as well as generative AI in the form of GPT-4o, showed a significant increase in performance when supplied with additional training data to learn from. Even when model training from scratch might be out of scope for a project, our experiments demonstrated that increasing the training samples for either model to 32, 64 or 128 would yield significant improvements over the previous step. In the long run, building a dedicated dataset and training a lighter, specialized model may not only yield superior accuracy and offer more transparency, but also prove to be significantly faster, more cost-effective and less ecologically harmful to run at scale.
            </p>
            <p>
               Importantly, researchers do not need to commit to a specific system before they begin annotating data. The creation of ground truth is a dynamic process. As we have demonstrated, evaluating multiple models can be beneficial even when only a small amount of training data is available. Comparing outputs across different training set sizes on multiple systems helps determine the utility of further manual annotation and may ultimately guide the final choice of system. Finally, we want to remind that the creation of data is a dynamic process. As we have shown and as should be considered by other approaches, the evaluation of multiple approaches is necessary, even if only a little training data is available. Comparing outputs with different amounts of training data can help to determine how useful the creation of more manually annotated data will be, and if resources should rather be put into the generation of annotated data, with the additional upside of creating something reusable.
            </p>
         </div>
         
         <div>
            <head>Acknowledgments</head>
            <p>This research was undertaken as part of the project <title rend="italic">Economies of Space</title>, which was funded by the Swiss National Science Foundation (Grant Nr. 208093). Calculations were performed on UBELIX (<ref target="https://www.id.unibe.ch/hpc">https://www.id.unibe.ch/hpc</ref>), the HPC cluster at the University of Bern.</p>
         </div>
        
         <div>
            <head>Appendix</head>
            <div>
               <head>Appendix A: Task 1 Prompt</head>
               <div> 
                 <head>### SETTING ###</head>
               <p>Als erfahrener Sprachwissenschaftler mit Spezialisierung auf Named Entity Recognition (NER) für historische Texte und Experte insbesondere für frühneuzeitliches Deutsch besteht deine Aufgabe darin, einen gegebenen Text für die maschinelle Weiterverwertung zu annotieren. Es handelt sich hierbei um einzelne Sätze, die zusammenhangslos transkribiert wurden. Dir wird in jeder Zeile ein Satz vorgegeben.
               </p>
               <p>Beachte, dass auch einzelne Abschnitte des Textes in Latein stehen können. Führe die NER dort dann auf Latein aus.
               </p>
               <p>Achte wirklich darauf, dass Du jeden Satz einzeln bearbeitest und zeichengenau wiedergibst. Nimm Dir Zeit und arbeite so genau wie möglich. Ich möchte die Ergebnisse später computationell weiterverarbeiten.
               </p>
               <p>Bei dem Text handelt es sich um einen Ausschnitt aus dem Historischen Grundbuch der Stadt Basel. Die Originaldokumente sind zwischen dem 14. und 18. Jahrhundert entstanden. Sie beschreiben Kaufverträge, Zinsverzeichnisse, Frönungen und weitere Ereignisse, die sich um Liegenschaften in der Stadt Basel drehen.
               </p>
              </div>
               
               <div>
                  <head>### INSTRUCTION ###</head>
                  <list type="ordered">
                     <head>Folge diesen Schritten:</head>
                     <item>Wiederhole den gegebenen Text exakt. Achte dabei sehr genau darauf, dass abgesehen von den Annotationen nichts hinzugefügt 
                        oder entfernt wird.</item>
                     <item>Die Aufgabe ist es BEREICHE bestimmten Kategorien zuzuordnen. Benutze hierfür Tags mit den entsprechenden Regeln: 
                        <list type="simple">
                           <item>- Wenn ein Bereich beginnt, markiere dies durch einen Kategorie-Tag. Zur verfügung stehen hier unter anderem die Kategorien '&lt;&lt;PER ' für Personen, '&lt;&lt;LOC ' für Orte und '&lt;&lt;ORG ' für Organisationen.</item>
                           <item>- Wenn ein Bereich endet, markiere das Ende durch den entsprechenden Tag, also ' /PER&gt;&gt;', ' /LOC&gt;&gt;' oder ' /ORG&gt;&gt;'.</item>
                           <item>- Achte darauf, dass jeder geöffnete Tag auch geschlossen werden muss.</item>
                           <item>- Tags dürfen sich überlappen oder geschachtelt sein - allerdings ist es unwahrscheinlich, dass dieses zu häufig passiert.</item>
                           <item>- BEACHTE: Die Texte sind in sehr alter Schrift gehalten und durch technische Verfahren fehlerbehaftet.</item>
                           <item>- BEACHTE: Früher hat man "nach Gehör" geschrieben. Dies bedeutet, dass für eine Erkennung ausreicht, wenn der Bereich sich *phonetisch* anhört wie eine Person, Ortschaft oder Organisation.</item>
                           <item>- BEACHTE: Teils handelt es sich um Reiseberichte mit altertümlichen Einheiten wie Meilen, alten Bezeichnungen für Orte, Gewässer und ähnliches.</item>
                           <item>- BEACHTE: Wir sind wirklich nur an Nennungen mit Eigennamen interessiert!</item>
                           <item>- BEACHTE: Bei geopolitischen Entitäten hängt die Kategorisierung vom Kontext ab. Wohnt jemand "zu Basel" ist es LOC, ist jemand ein "Bürger zu Basel" eine ORG.</item>
                           <item>- BEACHTE: Bei ORG und LOC Entitäten sind auch grammatikalische Kerne mitzuannotieren, sofern sie mit einem Eigennamen zusammenstehen. Es ist also z.B. "Kloster Klingental", nicht nur "Klingental".</item>
                           <item>- BEACHTE: Heiligennamen sind generell nicht als PER zu annotieren (sie kommen z.B. in Datumsangaben oder ORG-Namen vor).</item>
                        </list><!-- dash list inside 2 -->
                     </item>
                  </list><!-- numbered list -->
             </div>
              <div>
                 <head>### EXAMPLE ###</head>
              </div>
               <div>
                <head>### HINTS ###</head>
                  <list type="ordered">
                     <head>Beachte, wie in jedem Beispiel alle Schritte eingehalten wurden:</head>
                     <item>Wiederholung des exakten Textes.</item>
                     <item>Jeder Bereich hat einen Tag, der <list type="simple">
                        <item>- markiert ist</item>
                        <item>- Beginn und Ende hat</item>
                     </list></item>
                  </list>
               <p>Der Text muss ANSONSTEN unverändert reproduziert werden.
               </p>
               <p>Meine Karriere hängt davon ab, dass du keine Fehler machst. Daher bekommen sowohl du, als auch deine Oma jeweils $100k für jede erfolgreiche Annotation, die im weiteren Verlauf maschinenlesbar ist.
               </p>
               </div>
               <div>
                  <head>### TASK ###</head>
                  <list type="simple">
                     <head>Nun atme tief durch und gehe Schritt für Schritt vor. Hier der zu annotierende Text:</head>
                     <item>Role: user</item>
                     <item>```</item>
                     <item>INPUT_TEXT</item>
                     <item>```</item>
                  </list>
                  </div>
            </div><!-- end appendix a -->
            <div>
               <head>Appendix B: Task 2 Prompt (recursive)</head>
               <p>For Task 2 the prompt was modified depending on the annotation layer. This example shows the prompt for the document layer. First, an explainer added to the instructions which explained what type of textual element was to be annotated (reference, attribute, etc.). Second, the tagset was modified accordingly. For example, if the input was an entity reference, the instructions would be prefaced with <quote rend="inline"><hi rend="italic">Bei diesem Auszug spezifisch handelt es sich um eine Entitätenerwähnung. Dein Ziel ist es, die Elemente innerhalb dieser Erwähnung auszuzeichnen.</hi></quote>
               </p>
               <div>
                  <head>### SETTING ###</head>
               <p>Als erfahrener Sprachwissenschaftler mit Spezialisierung auf Named Entity Recognition (NER) für historische Texte und Experte insbesondere für frühneuzeitliches Deutsch besteht deine Aufgabe darin, einen gegebenen Text für die maschinelle Weiterverwertung zu annotieren. Es handelt sich hierbei um einzelne Sätze, die zusammenhangslos transkribiert wurden. Dir wird in jeder Zeile ein Satz vorgegeben.
               </p>
               <p>Beachte, dass auch einzelne Abschnitte des Textes in Latein stehen können. Führe die NER dort dann auf Latein aus.
               </p>
               <p>Achte wirklich darauf, dass Du jeden Satz einzeln bearbeitest und zeichengenau wiedergibst. Nimm Dir Zeit und arbeite so genau wie möglich. Ich möchte die Ergebnisse später computationell weiterverarbeiten.
               </p>
               <p>Bei dem Text handelt es sich um einen Ausschnitt aus dem Historischen Grundbuch der Stadt Basel. Die Originaldokumente sind zwischen dem 14. und 18. Jahrhundert entstanden. Sie beschreiben Kaufverträge, Zinsverzeichnisse, Frönungen und weitere Ereignisse, die sich um Liegenschaften in der Stadt Basel drehen.
               </p>
               </div><!-- setting end b -->
               <div>
                  <head>### INSTRUCTION ###</head>
                  <list type="ordered">
                     <head>Folge diesen Schritten:</head>
                     <item>Wiederhole den gegebenen Text exakt. Achte dabei sehr genau darauf, dass abgesehen von den Annotationen nichts
                        hinzugefügt oder entfernt wird.</item>
                     <item>Die Aufgabe ist es, BEREICHE bestimmten Kategorien zuzuordnen. Benutze hierfür Tags mit den entsprechenden Regeln:<list type="simple">
                        <item> - Wenn ein Bereich beginnt, markiere ihn nur einen <!-- ask Julia about arrows -->Tag. Verwende dafür das Format TAG. Welche Tags zur Auswahl stehen, siehst du im nächsten Punkt.</item>
                        <item>- Wenn ein Bereich endet, markiere das Ende durch den entsprechenden Tag, also /TAG>>.</item>
                        <item>- Achte darauf, dass jeder geöffnete Tag auch geschlossen werden muss.</item>
                        <item>- Falls sich Bereiche überlappen, markiere nur den äusseren Bereich.</item>
                     </list></item> <!-- die 2 instruction end -->
                     <item>Die zur Verfügung stehenden Tags sind:
                     <list type="simple">
                        <item>Referenzen auf Entitäten:<list type="simple">
                           <item>- PER für Personen</item>
                           <item>- LOC für Orte</item>
                           <item>- ORG für Organisationen</item>
                           <item>- GPE für Geopolitische Entitäten mit einer Unterklasse:<list type="simple">
                              <item>- GPE-GPE ohne oder in unklarem Kontext</item>
                              <item>- GPE-ORG im Kontext einer Organisation</item>
                              <item>- GPE-LOC im Kontext eines Ortes</item>
                           </list></item><!-- 2sub 3 instruction end -->
                        </list></item><!-- sub 3 instruction end -->
                       <item>Werte:<list type="simple">
                          <item>- DATE für Datumsangaben</item>
                          <item>- TIME für Zeitangaben</item>
                          <item>- CURR für Geldwerte oder andere Zahlungsmittel</item>
                       </list></item><!-- werte sub end -->
                        <item>Ereignisse:<list type="simple">
                           <item>- EV für Triggerwörter für Ereignisse mit einer Unterklasse, welche das Ereignis</item> 
                        </list></item><!-- Ereignisse sub end -->
                        <item>bestimmt: <list type="simple">
                          <item> EV-ACQUISITION</item>
                           <item>EV-CONSENT</item>
                           <item>EV-CONSTRUCTION</item>
                          <item> EV-DEBT</item>
                          <item> EV-DECLARATION</item>
                           <item>EV-DUE</item>
                           <item>EV-ENDOWMENT</item>
                          <item> EV-EXCHANGE</item>
                         <item>EV-GIFT</item>
                          <item> EV-HEREDITARY</item>
                           <item>EV-INHERITANCE</item>
                           <item>EV-LITIGATION</item>
                           <item>EV-OTHER</item>
                           <item>EV-OWNERSHIP</item>
                           <item>EV-PAYMENT</item>
                           <item>EV-REDEMPTION</item>
                          <item> EV-RENT</item>
                          <item> EV-REPOSSESSION</item>
                           <item>EV-SALE</item>
                          <item> EV-SANCTION</item>
                           <item>EV-SEIZURE</item>
                          <item> EV-TESTAMENT</item>
                        <item>EV-TRANSFER</item>
                         <item>EV-TYPE</item>
                        </list></item>
                        <item>Diese Annotation ist recht komplex, orientiere dich daher an den gegebenen Beispielen.</item>
                     </list></item><!-- die 3 instruction end -->
                  </list><!-- instruction b end -->
                  <list type="simple">
                    <item> - BEACHTE: Die Texte sind in sehr alter Schrift gehalten und durch technische 
                       Verfahren fehlerbehaftet.</item>
                     <item>- BEACHTE: Früher hat man "nach Gehör" geschrieben. Dies bedeutet, dass für eine 
                        Erkennung ausreicht, wenn der Bereich sich *phonetisch* anhört wie eine Person, Ortschaft oder Organisation.</item>
                     <item>- BEACHTE: Teils handelt es sich um Reiseberichte mit altertümlichen Einheiten wie Meilen, 
                        alten Bezeichnungen für Orte, Gewässer und ähnliches.</item>
                  </list>
               </div>
               <div>
                  <head>### EXAMPLE ###</head>
               </div>
               <div>
                  <head>### HINTS ###</head>
                  <list type="simple">
                     <head>Beachte, wie in jedem Beispiel alle Schritte eingehalten wurden:</head>
                     <item>1. Wiederholung des exakten Textes.</item>
                     <item>2. Jeder Bereich hat einen Tag, der<list type="simple">
                        <item>- markiert ist</item>
                        <item>- Beginn und Ende hat</item>
                     </list></item>
                     <item>Der Text muss ANSONSTEN unverändert reproduziert werden.</item>
                     <item>Meine Karriere hängt davon ab, dass du keine Fehler machst. Daher bekommen sowohl du, als auch 
                        deine Oma jeweils $100k für jede erfolgreiche Annotation, die im weiteren Verlauf maschinenlesbar ist.</item>
                  </list>
               </div>
               <div>
                  <head>### TASK ###</head>
                  <list type="simple">
                     <item>Nun atme tief durch und gehe Schritt für Schritt vor. Hier der zu annotierende Text:<list type="simple">
                        <item>Role: user</item>
                        <item>```</item>
                     </list>INPUT_TEXT</item>
                     <item> ```</item>
                  </list>
               </div>
            </div><!-- appendix b end -->
            <div>
               <head>Appendix C: Task 2 Prompt (single)</head>
               <div>
                  <head>### SETTING ###</head>
               <p>Als erfahrener Sprachwissenschaftler mit Spezialisierung auf Named Entity Recognition (NER) für historische Texte und Experte insbesondere für frühneuzeitliches Deutsch besteht deine Aufgabe darin, einen gegebenen Text für die maschinelle Weiterverwertung zu annotieren. 
                  Es handelt sich hierbei um einzelne Sätze, die zusammenhangslos transkribiert wurden. Dir wird in jeder Zeile ein Satz vorgegeben.
               </p>
               <p>Beachte, dass auch einzelne Abschnitte des Textes in Latein stehen können. Führe die NER dort dann auf Latein aus.
               </p>
               <p>Achte wirklich darauf, dass Du jeden Satz einzeln bearbeitest und zeichengenau wiedergibst. Nimm Dir Zeit und arbeite so genau wie möglich. Ich möchte die Ergebnisse später computationell weiterverarbeiten.
               </p>
               <p>Bei dem Text handelt es sich um einen Ausschnitt aus dem Historischen Grundbuch der Stadt Basel. Die Originaldokumente sind zwischen dem 14. und 18. Jahrhundert entstanden. Sie beschreiben Kaufverträge, Zinsverzeichnisse, Frönungen und weitere Ereignisse, die sich um Liegenschaften in der Stadt Basel drehen.
               </p>
               </div>
               <div>
                  <head>### INSTRUCTION ###</head>
               <list type="simple">
                  <head>Folge diesen Schritten:</head>
               <item>1. Wiederhole den gegebenen Text exakt. Achte dabei sehr genau darauf, dass abgesehen von den 
                  Annotationen nichts hinzugefügt oder entfernt wird.</item>
               <item>2. Die Aufgabe ist es BEREICHE bestimmten Kategorien zuzuordnen. Benutze hierfür Tags mit den entsprechenden Regeln:<list type="simple">
                  <item>- Wenn ein Bereich beginnt, markiere ihn nur einen Tag. Verwende dafür das Format &lt;&lt;TAG. Welche Tags zur Auswahl stehen siehst du im nächsten Puntk.</item>
                  <item>- Wenn ein Bereich endet, markiere das Ende durch den entsprechenden Tag, also /TAG&gt;&gt;.</item>
                  <item>- Achte darauf, dass jeder geöffnete Tag auch geschlossen werden muss.</item>
                  <item>- Tags dürfen verschachtelt sein, und sind es auch häufig.</item>
               </list></item>
                  <item>3. Die zur Verfügung stehenden Tags sind:<list type="simple">
                     <item>Referenzen auf Entitäten:<list type="simple">
                        <item>- PER für Personen</item>
                        <item>- LOC für Orte</item>
                        <item>- ORG für Organisationen</item>
                     </list></item>
                     <item>Beschreibende Elemente:<list type="simple">
                        <item>- HEAD für den grammatikalischen Kern</item>
                        <item>- APPO für Appositionen</item>
                        <item>- ATTR für Attribute</item>
                     </list></item>
                     <item>Werte:<list type="simple">
                        <item>- DATE für Datumsangaben</item>
                        <item>- CURR für Geldwerte oder andere Zahlungsmittel</item>
                     </list></item>
                  </list></item>
                  <item>Diese Annotation ist recht komplex, orientiere dich daher an den gegebenen Beispielen.</item>
               </list>
                  <list type="simple">
                     <item>- BEACHTE: Die Texte sind in sehr alter Schrift gehalten und durch technische Verfahren fehlerbehaftet.</item>
                     <item>- BEACHTE: Früher hat man "nach Gehör" geschrieben. Dies bedeutet, dass für eine Erkennung ausreicht, wenn der Bereich sich *phonetisch* anhört wie eine Person, Ortschaft oder Organisation.</item>
                     <item>- BEACHTE: Teils handelt es sich um Reiseberichte mit altertümlichen Einheiten wie Meilen, alten Bezeichnungen für Orte, Gewässer und ähnliches.</item>
                     <item>- BEACHTE: Referenzen auf Entitäten können sehr lange Spannen umfassen, da auch Artikel und Attribute mit annotiert werden sollen.</item>
                     <item>- BEACHTE: Eigennamen sind keine Voraussetzung für eine Annotation. Sämtliche Bezeichnungen, die eindeutig auf eine Person, einen Ort oder eine Organisation verweisen, sind zu markieren.</item>
                     <item>- BEACHTE: Beschreibende Elemente (HEAD, APPO, ATTR) können nie ausserhalb von Referenzen auf Entitäten (PER, LOC, ORG) oder Appositionen (APPO) auftreten.</item>
                     <item>- BEACHTE: Bei geopolitischen Entitäten hängt die Kategorisierung vom Kontext ab. Wohnt jemand "zu Basel" ist es LOC, ist jemand ein "Bürger zu Basel" eine ORG.</item>
                     <item>- BEACHTE: Heiligennamen sind generell nicht als PER zu annotieren (sie kommen z.B. in Datumsangaben oder ORG-Namen vor).</item>
                     <item>- BEACHTE: Die Grenzen der Bereiche folgen meist den Regeln der Konstituenzgrammatik (Referenzen =&gt; Nominalphrase, HEAD =&gt; Kern). Es kann aber auch vorkommen, dass diese davon abweichen.</item>
                  </list>
               
               <div>
                  <head>### EXAMPLE ###</head>
               </div>
               <div>
                  <head>### HINTS ###</head>
                  <list type="simple">
                     <head>Beachte, wie in jedem Beispiel alle Schritte einghalten wurden:</head>
                     <item>1. Wiederholung des exakten Textes.</item>
                     <item>2. Jeder Bereich hat einen Tag, der<list type="simple">
                        <item>- markiert ist</item>
                        <item>- Beginn und Ende hat</item>
                        <item></item>
                     </list></item>
                  </list>
                  
                  <p>Der Text muss ANSONSTEN unverändert reproduziert werden.</p>
                  
                  <p>Meine Karriere hängt davon ab, dass du keine Fehler machst. 
                     Daher bekommmen sowohl du, als auch deine Oma jeweils $100k für jede erfolgreiche annotation, die im weiteren Verlauf maschienenlesbar ist.</p>
               </div>
              <div>
                 <head>### TASK ###</head>
                 <list type="simple">
                    <head>Nun atme tief durch und gehe Schritt für Schritt vor. Hier der zu annotierende Text:</head>
                    <item>Role: user</item>
                    <item>```</item>
                    <item>INPUT_TEXT</item>
                    <item>```</item>
                 </list>
              </div>
            </div>
            </div>
            <div>
               <head>Appendix D: Task 1 Results</head>
               <p>Note that we performed two more runs for flair-custom-128-samples because the variance between the initial three runs was extraordinarily high.</p>
               <table>
                  <row role="data">
                     <cell>approach</cell>
                     <cell>ablation</cell>
                     <cell>run</cell>
                     <cell>f1 score</cell>
                  </row>
                  <row role="data">
                     <cell>flair-custom</cell>
                     <cell>16</cell>
                     <cell>0</cell>
                     <cell>59.23%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-custom</cell>
                     <cell>16</cell>
                     <cell>1</cell>
                     <cell>62.19%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-custom</cell>
                     <cell>16</cell>
                     <cell>2</cell>
                     <cell>59.61%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-custom</cell>
                     <cell>32</cell>
                     <cell>0</cell>
                     <cell>72.34%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-custom</cell>
                     <cell>32</cell>
                     <cell>1</cell>
                     <cell>63.69%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-custom</cell>
                     <cell>32</cell>
                     <cell>2</cell>
                     <cell>70.46%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-custom</cell>
                     <cell>64</cell>
                     <cell>0</cell>
                     <cell>70.48%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-custom</cell>
                     <cell>64</cell>
                     <cell>1</cell>
                     <cell>73.55%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-custom</cell>
                     <cell>64</cell>
                     <cell>2</cell>
                     <cell>75.62%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-custom</cell>
                     <cell>128</cell>
                     <cell>0</cell>
                     <cell>72.49%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-custom</cell>
                     <cell>128</cell>
                     <cell>1</cell>
                     <cell>79.70%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-custom</cell>
                     <cell>128</cell>
                     <cell>2</cell>
                     <cell>79.20%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-custom</cell>
                     <cell>128</cell>
                     <cell>3</cell>
                     <cell>77.14%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-custom</cell>
                     <cell>128</cell>
                     <cell>4</cell>
                     <cell>74.98%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-custom</cell>
                     <cell>128</cell>
                     <cell>5</cell>
                     <cell>76.32%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-custom</cell>
                     <cell>256</cell>
                     <cell>0</cell>
                     <cell>83.37%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-custom</cell>
                     <cell>256</cell>
                     <cell>1</cell>
                     <cell>83.74%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-custom</cell>
                     <cell>256</cell>
                     <cell>2</cell>
                     <cell>81.43%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-custom</cell>
                     <cell>512</cell>
                     <cell>0</cell>
                     <cell>85.46%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-custom</cell>
                     <cell>512</cell>
                     <cell>1</cell>
                     <cell>85.48%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-custom</cell>
                     <cell>512</cell>
                     <cell>2</cell>
                     <cell>85.63%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-custom</cell>
                     <cell>745</cell>
                     <cell>0</cell>
                     <cell>86.42%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-custom</cell>
                     <cell>745</cell>
                     <cell>1</cell>
                     <cell>85.99%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-custom</cell>
                     <cell>745</cell>
                     <cell>2</cell>
                     <cell>85.20%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o random</cell>
                     <cell>4</cell>
                     <cell>0</cell>
                     <cell>64.68%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o random</cell>
                     <cell>4</cell>
                     <cell>1</cell>
                     <cell>71.41%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o random</cell>
                     <cell>4</cell>
                     <cell>2</cell>
                     <cell>66.37%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o random</cell>
                     <cell>8</cell>
                     <cell>0</cell>
                     <cell>69.77%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o random</cell>
                     <cell>8</cell>
                     <cell>1</cell>
                     <cell>62.85%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o random</cell>
                     <cell>8</cell>
                     <cell>2</cell>
                     <cell>71.03%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o random</cell>
                     <cell>16</cell>
                     <cell>0</cell>
                     <cell>71.95%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o random</cell>
                     <cell>16</cell>
                     <cell>1</cell>
                     <cell>68.13%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o random</cell>
                     <cell>16</cell>
                     <cell>2</cell>
                     <cell>66.27%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o random</cell>
                     <cell>32</cell>
                     <cell>0</cell>
                     <cell>74.09%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o random</cell>
                     <cell>32</cell>
                     <cell>1</cell>
                     <cell>75.73%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o random</cell>
                     <cell>32</cell>
                     <cell>2</cell>
                     <cell>73.72%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o random</cell>
                     <cell>64</cell>
                     <cell>0</cell>
                     <cell>75.93%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o random</cell>
                     <cell>64</cell>
                     <cell>1</cell>
                     <cell>77.36%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o random</cell>
                     <cell>64</cell>
                     <cell>2</cell>
                     <cell>77.28%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o random</cell>
                     <cell>128</cell>
                     <cell>0</cell>
                     <cell>78.53%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o random</cell>
                     <cell>128</cell>
                     <cell>1</cell>
                     <cell>76.84%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o random</cell>
                     <cell>128</cell>
                     <cell>2</cell>
                     <cell>77.78%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o similarity</cell>
                     <cell>256</cell>
                     <cell>0</cell>
                     <cell>79.14%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o similarity</cell>
                     <cell>256</cell>
                     <cell>1</cell>
                     <cell>80.95%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o similarity</cell>
                     <cell>256</cell>
                     <cell>2</cell>
                     <cell>80.06%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o similarity</cell>
                     <cell>512</cell>
                     <cell>0</cell>
                     <cell>80.18%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o similarity</cell>
                     <cell>512</cell>
                     <cell>1</cell>
                     <cell>79.47%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o similarity</cell>
                     <cell>512</cell>
                     <cell>2</cell>
                     <cell>80.95%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o similarity</cell>
                     <cell>662</cell>
                     <cell>0</cell>
                     <cell>80.89%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o similarity</cell>
                     <cell>662</cell>
                     <cell>1</cell>
                     <cell>82.42%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o similarity</cell>
                     <cell>662</cell>
                     <cell>2</cell>
                     <cell>81.92%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-finetuned</cell>
                     <cell>16</cell>
                     <cell>0</cell>
                     <cell>50.97%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-finetuned</cell>
                     <cell>16</cell>
                     <cell>1</cell>
                     <cell>49.36%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-finetuned</cell>
                     <cell>16</cell>
                     <cell>2</cell>
                     <cell>51.50%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-finetuned</cell>
                     <cell>32</cell>
                     <cell>0</cell>
                     <cell>55.39%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-finetuned</cell>
                     <cell>32</cell>
                     <cell>1</cell>
                     <cell>56.88%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-finetuned</cell>
                     <cell>32</cell>
                     <cell>2</cell>
                     <cell>57.26%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-finetuned</cell>
                     <cell>64</cell>
                     <cell>0</cell>
                     <cell>62.54%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-finetuned</cell>
                     <cell>64</cell>
                     <cell>1</cell>
                     <cell>62.50%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-finetuned</cell>
                     <cell>64</cell>
                     <cell>2</cell>
                     <cell>63.67%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-finetuned</cell>
                     <cell>128</cell>
                     <cell>0</cell>
                     <cell>67.72%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-finetuned</cell>
                     <cell>128</cell>
                     <cell>1</cell>
                     <cell>65.93%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-finetuned</cell>
                     <cell>128</cell>
                     <cell>2</cell>
                     <cell>66.98%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-finetuned</cell>
                     <cell>256</cell>
                     <cell>0</cell>
                     <cell>72.12%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-finetuned</cell>
                     <cell>256</cell>
                     <cell>1</cell>
                     <cell>72.34%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-finetuned</cell>
                     <cell>256</cell>
                     <cell>2</cell>
                     <cell>71.77%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-finetuned</cell>
                     <cell>512</cell>
                     <cell>0</cell>
                     <cell>75.02%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-finetuned</cell>
                     <cell>512</cell>
                     <cell>1</cell>
                     <cell>75.76%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-finetuned</cell>
                     <cell>512</cell>
                     <cell>2</cell>
                     <cell>75.61%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-finetuned</cell>
                     <cell>745</cell>
                     <cell>0</cell>
                     <cell>78.97%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-finetuned</cell>
                     <cell>745</cell>
                     <cell>1</cell>
                     <cell>77.64%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-finetuned</cell>
                     <cell>745</cell>
                     <cell>2</cell>
                     <cell>78.00%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-ner-german</cell>
                     <cell>0</cell>
                     <cell>0</cell>
                     <cell>40.54%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-ner-german</cell>
                     <cell>0</cell>
                     <cell>1</cell>
                     <cell>46.18%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-ner-german</cell>
                     <cell>0</cell>
                     <cell>2</cell>
                     <cell>43.42%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-ner-german</cell>
                     <cell>745</cell>
                     <cell>0</cell>
                     <cell>40.54%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-ner-german</cell>
                     <cell>745</cell>
                     <cell>1</cell>
                     <cell>46.18%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-ner-german</cell>
                     <cell>745</cell>
                     <cell>2</cell>
                     <cell>43.42%</cell>
                  </row>
               </table>
            </div>
            <div>
               <head>Appendix E: Element + Class Tagset for Task 2</head>
               <table>
                  <row role="data">
                     <cell>tag</cell>
                     <cell>frequency in training set</cell>
                  </row>
                  <row role="data">
                     <cell>head-nam</cell>
                     <cell>4027</cell>
                  </row>
                  <row role="data">
                     <cell>per</cell>
                     <cell>2874</cell>
                  </row>
                  <row role="data">
                     <cell>date</cell>
                     <cell>2335</cell>
                  </row>
                  <row role="data">
                     <cell>fac</cell>
                     <cell>2086</cell>
                  </row>
                  <row role="data">
                     <cell>head-type</cell>
                     <cell>1425</cell>
                  </row>
                  <row role="data">
                     <cell>curr</cell>
                     <cell>1353</cell>
                  </row>
                  <row role="data">
                     <cell>attr-loc</cell>
                     <cell>821</cell>
                  </row>
                  <row role="data">
                     <cell>head-occ</cell>
                     <cell>813</cell>
                  </row>
                  <row role="data">
                     <cell>appo-occ</cell>
                     <cell>736</cell>
                  </row>
                  <row role="data">
                     <cell>head-pro</cell>
                     <cell>705</cell>
                  </row>
                  <row role="data">
                     <cell>ev-due</cell>
                     <cell>645</cell>
                  </row>
                  <row role="data">
                     <cell>org</cell>
                     <cell>520</cell>
                  </row>
                  <row role="data">
                     <cell>attr-owner</cell>
                     <cell>518</cell>
                  </row>
                  <row role="data">
                     <cell>head-fam</cell>
                     <cell>357</cell>
                  </row>
                  <row role="data">
                     <cell>appo-fam</cell>
                     <cell>282</cell>
                  </row>
                  <row role="data">
                     <cell>loc</cell>
                     <cell>261</cell>
                  </row>
                  <row role="data">
                     <cell>appo-nam</cell>
                     <cell>261</cell>
                  </row>
                  <row role="data">
                     <cell>gpe-loc</cell>
                     <cell>238</cell>
                  </row>
                  <row role="data">
                     <cell>ev-sale</cell>
                     <cell>154</cell>
                  </row>
                  <row role="data">
                     <cell>attr-dead</cell>
                     <cell>153</cell>
                  </row>
                  <row role="data">
                     <cell>attr-dues</cell>
                     <cell>139</cell>
                  </row>
                  <row role="data">
                     <cell>appo-title</cell>
                     <cell>127</cell>
                  </row>
                  <row role="data">
                     <cell>head-title</cell>
                     <cell>127</cell>
                  </row>
                  <row role="data">
                     <cell>gpe-org</cell>
                     <cell>107</cell>
                  </row>
                  <row role="data">
                     <cell>attr-includes</cell>
                     <cell>104</cell>
                  </row>
                  <row role="data">
                     <cell>ev-type</cell>
                     <cell>103</cell>
                  </row>
                  <row role="data">
                     <cell>head-org-aff</cell>
                     <cell>97</cell>
                  </row>
                  <row role="data">
                     <cell>appo-org-aff</cell>
                     <cell>95</cell>
                  </row>
                  <row role="data">
                     <cell>ev-seizure</cell>
                     <cell>84</cell>
                  </row>
                  <row role="data">
                     <cell>head-heir</cell>
                     <cell>66</cell>
                  </row>
                  <row role="data">
                     <cell>ev-rent</cell>
                     <cell>65</cell>
                  </row>
                  <row role="data">
                     <cell>head-role</cell>
                     <cell>63</cell>
                  </row>
                  <row role="data">
                     <cell>attr-org-aff</cell>
                     <cell>40</cell>
                  </row>
                  <row role="data">
                     <cell>ev-litigation</cell>
                     <cell>38</cell>
                  </row>
                  <row role="data">
                     <cell>head-owner</cell>
                     <cell>29</cell>
                  </row>
                  <row role="data">
                     <cell>head-gen</cell>
                     <cell>20</cell>
                  </row>
                  <row role="data">
                     <cell>appo-gen</cell>
                     <cell>20</cell>
                  </row>
                  <row role="data">
                     <cell>ev-redemption</cell>
                     <cell>18</cell>
                  </row>
                  <row role="data">
                     <cell>appo-owner</cell>
                     <cell>17</cell>
                  </row>
                  <row role="data">
                     <cell>ev-testament</cell>
                     <cell>15</cell>
                  </row>
                  <row role="data">
                     <cell>unk</cell>
                     <cell>15</cell>
                  </row>
                  <row role="data">
                     <cell>ev-hereditary</cell>
                     <cell>14</cell>
                  </row>
                  <row role="data">
                     <cell>appo-role</cell>
                     <cell>12</cell>
                  </row>
                  <row role="data">
                     <cell>gpe-gpe</cell>
                     <cell>12</cell>
                  </row>
                  <row role="data">
                     <cell>ev-payment</cell>
                     <cell>11</cell>
                  </row>
                  <row role="data">
                     <cell>head-repr</cell>
                     <cell>11</cell>
                  </row>
                  <row role="data">
                     <cell>ev-consent</cell>
                     <cell>11</cell>
                  </row>
                  <row role="data">
                     <cell>attr-part-of</cell>
                     <cell>11</cell>
                  </row>
                  <row role="data">
                     <cell>ev-declaration</cell>
                     <cell>10</cell>
                  </row>
                  <row role="data">
                     <cell>ev-transfer</cell>
                     <cell>10</cell>
                  </row>
                  <row role="data">
                     <cell>head-self</cell>
                     <cell>10</cell>
                  </row>
                  <row role="data">
                     <cell>ev-bequest</cell>
                     <cell>9</cell>
                  </row>
                  <row role="data">
                     <cell>head-unk</cell>
                     <cell>9</cell>
                  </row>
                  <row role="data">
                     <cell>ev-inheritance</cell>
                     <cell>8</cell>
                  </row>
                  <row role="data">
                     <cell>head-rel</cell>
                     <cell>8</cell>
                  </row>
                  <row role="data">
                     <cell>ev-construction</cell>
                     <cell>7</cell>
                  </row>
                  <row role="data">
                     <cell>ev-pledge</cell>
                     <cell>6</cell>
                  </row>
                  <row role="data">
                     <cell>ev-offer</cell>
                     <cell>5</cell>
                  </row>
                  <row role="data">
                     <cell>head-creditor</cell>
                     <cell>5</cell>
                  </row>
                  <row role="data">
                     <cell>appo-rel</cell>
                     <cell>5</cell>
                  </row>
                  <row role="data">
                     <cell>appo-unk</cell>
                     <cell>5</cell>
                  </row>
                  <row role="data">
                     <cell>appo-type</cell>
                     <cell>5</cell>
                  </row>
                  <row role="data">
                     <cell>attr-unk</cell>
                     <cell>5</cell>
                  </row>
                  <row role="data">
                     <cell>attr-type</cell>
                     <cell>5</cell>
                  </row>
                  <row role="data">
                     <cell>ev-sanction</cell>
                     <cell>5</cell>
                  </row>
                  <row role="data">
                     <cell>head-quant</cell>
                     <cell>5</cell>
                  </row>
                  <row role="data">
                     <cell>attr-sale</cell>
                     <cell>4</cell>
                  </row>
                  <row role="data">
                     <cell>attr-other</cell>
                     <cell>4</cell>
                  </row>
                  <row role="data">
                     <cell>head-loc</cell>
                     <cell>4</cell>
                  </row>
                  <row role="data">
                     <cell>attr-has-member</cell>
                     <cell>3</cell>
                  </row>
                  <row role="data">
                     <cell>ev-endowment</cell>
                     <cell>3</cell>
                  </row>
                  <row role="data">
                     <cell>ev-outbid</cell>
                     <cell>3</cell>
                  </row>
                  <row role="data">
                     <cell>unclear-unclear</cell>
                     <cell>3</cell>
                  </row>
                  <row role="data">
                     <cell>appo-repr</cell>
                     <cell>2</cell>
                  </row>
                  <row role="data">
                     <cell>ev-debt</cell>
                     <cell>2</cell>
                  </row>
                  <row role="data">
                     <cell>ev-other</cell>
                     <cell>2</cell>
                  </row>
                  <row role="data">
                     <cell>appo-loc</cell>
                     <cell>1</cell>
                  </row>
                  <row role="data">
                     <cell>appo-creditor</cell>
                     <cell>1</cell>
                  </row>
                  <row role="data">
                     <cell>attr-creditor</cell>
                     <cell>1</cell>
                  </row>
                  <row role="data">
                     <cell>ev-receipt</cell>
                     <cell>1</cell>
                  </row>
               </table>
            </div>
            <div>
               <head>Appendix F: Task 2 Results</head>
               <table>
                  <row role="data">
                     <cell>approach</cell>
                     <cell>ablation</cell>
                     <cell>run</cell>
                     <cell>f1 score</cell>
                  </row>
                  <row role="data">
                     <cell>flair-custom</cell>
                     <cell>8</cell>
                     <cell>0</cell>
                     <cell>24.44%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-custom</cell>
                     <cell>8</cell>
                     <cell>1</cell>
                     <cell>31.40%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-custom</cell>
                     <cell>8</cell>
                     <cell>2</cell>
                     <cell>36.48%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-custom</cell>
                     <cell>16</cell>
                     <cell>0</cell>
                     <cell>35.92%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-custom</cell>
                     <cell>16</cell>
                     <cell>1</cell>
                     <cell>36.52%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-custom</cell>
                     <cell>16</cell>
                     <cell>2</cell>
                     <cell>45.98%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-custom</cell>
                     <cell>32</cell>
                     <cell>0</cell>
                     <cell>55.03%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-custom</cell>
                     <cell>32</cell>
                     <cell>1</cell>
                     <cell>52.53%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-custom</cell>
                     <cell>32</cell>
                     <cell>2</cell>
                     <cell>56.67%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-custom</cell>
                     <cell>64</cell>
                     <cell>0</cell>
                     <cell>67.64%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-custom</cell>
                     <cell>64</cell>
                     <cell>1</cell>
                     <cell>68.47%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-custom</cell>
                     <cell>64</cell>
                     <cell>2</cell>
                     <cell>65.98%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-custom</cell>
                     <cell>128</cell>
                     <cell>0</cell>
                     <cell>71.51%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-custom</cell>
                     <cell>128</cell>
                     <cell>1</cell>
                     <cell>72.21%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-custom</cell>
                     <cell>128</cell>
                     <cell>2</cell>
                     <cell>70.64%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-custom</cell>
                     <cell>256</cell>
                     <cell>0</cell>
                     <cell>77.28%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-custom</cell>
                     <cell>256</cell>
                     <cell>1</cell>
                     <cell>76.21%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-custom</cell>
                     <cell>256</cell>
                     <cell>2</cell>
                     <cell>75.86%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-custom</cell>
                     <cell>512</cell>
                     <cell>0</cell>
                     <cell>79.22%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-custom</cell>
                     <cell>512</cell>
                     <cell>1</cell>
                     <cell>80.86%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-custom</cell>
                     <cell>512</cell>
                     <cell>2</cell>
                     <cell>79.89%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-custom</cell>
                     <cell>745</cell>
                     <cell>0</cell>
                     <cell>82.28%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-custom</cell>
                     <cell>745</cell>
                     <cell>1</cell>
                     <cell>82.15%</cell>
                  </row>
                  <row role="data">
                     <cell>flair-custom</cell>
                     <cell>745</cell>
                     <cell>2</cell>
                     <cell>81.65%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o similarity recursive</cell>
                     <cell>8</cell>
                     <cell>0</cell>
                     <cell>53.61%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o similarity recursive</cell>
                     <cell>8</cell>
                     <cell>1</cell>
                     <cell>58.06%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o similarity recursive</cell>
                     <cell>8</cell>
                     <cell>2</cell>
                     <cell>52.58%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o similarity recursive</cell>
                     <cell>16</cell>
                     <cell>0</cell>
                     <cell>57.53%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o similarity recursive</cell>
                     <cell>16</cell>
                     <cell>1</cell>
                     <cell>57.72%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o similarity recursive</cell>
                     <cell>16</cell>
                     <cell>2</cell>
                     <cell>57.36%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o similarity recursive</cell>
                     <cell>32</cell>
                     <cell>0</cell>
                     <cell>63.87%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o similarity recursive</cell>
                     <cell>32</cell>
                     <cell>1</cell>
                     <cell>63.84%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o similarity recursive</cell>
                     <cell>32</cell>
                     <cell>2</cell>
                     <cell>60.79%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o similarity recursive</cell>
                     <cell>64</cell>
                     <cell>0</cell>
                     <cell>67.80%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o similarity recursive</cell>
                     <cell>64</cell>
                     <cell>1</cell>
                     <cell>65.71%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o similarity recursive</cell>
                     <cell>64</cell>
                     <cell>2</cell>
                     <cell>66.73%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o similarity recursive</cell>
                     <cell>128</cell>
                     <cell>0</cell>
                     <cell>68.76%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o similarity recursive</cell>
                     <cell>128</cell>
                     <cell>1</cell>
                     <cell>71.23%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o similarity recursive</cell>
                     <cell>128</cell>
                     <cell>2</cell>
                     <cell>70.60%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o similarity recursive</cell>
                     <cell>256</cell>
                     <cell>0</cell>
                     <cell>73.58%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o similarity recursive</cell>
                     <cell>256</cell>
                     <cell>1</cell>
                     <cell>73.10%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o similarity recursive</cell>
                     <cell>256</cell>
                     <cell>2</cell>
                     <cell>73.87%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o similarity recursive</cell>
                     <cell>512</cell>
                     <cell>0</cell>
                     <cell>76.26%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o similarity recursive</cell>
                     <cell>512</cell>
                     <cell>1</cell>
                     <cell>74.30%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o similarity recursive</cell>
                     <cell>512</cell>
                     <cell>2</cell>
                     <cell>75.30%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o similarity recursive</cell>
                     <cell>662</cell>
                     <cell>0</cell>
                     <cell>75.86%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o similarity recursive</cell>
                     <cell>662</cell>
                     <cell>1</cell>
                     <cell>75.82%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o similarity recursive</cell>
                     <cell>662</cell>
                     <cell>2</cell>
                     <cell>75.45%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o similarity single prompt</cell>
                     <cell>8</cell>
                     <cell>0</cell>
                     <cell>49.22%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o similarity single prompt</cell>
                     <cell>8</cell>
                     <cell>1</cell>
                     <cell>49.10%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o similarity single prompt</cell>
                     <cell>8</cell>
                     <cell>2</cell>
                     <cell>48.53%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o similarity single prompt</cell>
                     <cell>16</cell>
                     <cell>0</cell>
                     <cell>55.95%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o similarity single prompt</cell>
                     <cell>16</cell>
                     <cell>1</cell>
                     <cell>55.27%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o similarity single prompt</cell>
                     <cell>16</cell>
                     <cell>2</cell>
                     <cell>58.77%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o similarity single prompt</cell>
                     <cell>32</cell>
                     <cell>0</cell>
                     <cell>57.99%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o similarity single prompt</cell>
                     <cell>32</cell>
                     <cell>1</cell>
                     <cell>60.29%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o similarity single prompt</cell>
                     <cell>32</cell>
                     <cell>2</cell>
                     <cell>60.22%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o similarity single prompt</cell>
                     <cell>64</cell>
                     <cell>0</cell>
                     <cell>62.82%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o similarity single prompt</cell>
                     <cell>64</cell>
                     <cell>1</cell>
                     <cell>64.24%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o similarity single prompt</cell>
                     <cell>64</cell>
                     <cell>2</cell>
                     <cell>62.88%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o similarity single prompt</cell>
                     <cell>128</cell>
                     <cell>0</cell>
                     <cell>65.81%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o similarity single prompt</cell>
                     <cell>128</cell>
                     <cell>1</cell>
                     <cell>65.12%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o similarity single prompt</cell>
                     <cell>128</cell>
                     <cell>2</cell>
                     <cell>64.34%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o similarity single prompt</cell>
                     <cell>256</cell>
                     <cell>0</cell>
                     <cell>66.73%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o similarity single prompt</cell>
                     <cell>256</cell>
                     <cell>1</cell>
                     <cell>67.54%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o similarity single prompt</cell>
                     <cell>256</cell>
                     <cell>2</cell>
                     <cell>67.71%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o similarity single prompt</cell>
                     <cell>512</cell>
                     <cell>0</cell>
                     <cell>67.85%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o similarity single prompt</cell>
                     <cell>512</cell>
                     <cell>1</cell>
                     <cell>67.14%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o similarity single prompt</cell>
                     <cell>512</cell>
                     <cell>2</cell>
                     <cell>68.82%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o similarity single prompt</cell>
                     <cell>662</cell>
                     <cell>0</cell>
                     <cell>68.91%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o similarity single prompt</cell>
                     <cell>662</cell>
                     <cell>1</cell>
                     <cell>68.30%</cell>
                  </row>
                  <row role="data">
                     <cell>gpt-4o similarity single prompt</cell>
                     <cell>662</cell>
                     <cell>2</cell>
                     <cell>69.33%</cell>
                  </row>
               </table>
            </div>
         </div>
        
      </body>
      <back>
         <listBibl>
            <bibl xml:id="aguilar2022" label="Aguilar 2022">Aguilar, S. T. (2022) <title rend="quotes">Multilingual named entity recognition for medieval charters using stacked embeddings and bert-based models</title>, in <title rend="italic">Proceedings of the Second Workshop on Language Technologies for Historical and Ancient Languages</title>, pp. 119–128.</bibl>
            
            <bibl xml:id="akbik2019" label="Akbik et al. 2019">Akbik, A. et al. (2019) <title rend="quotes">FLAIR: An easy-to-use framework for state-of-the-art NLP</title>, in <title rend="italic">Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics (Demonstrations)</title>, pp. 54–59.</bibl>
            
            <bibl xml:id="akbik2018" label="Akbik, Blythe, and Vollgraf 2018">Akbik, A., Blythe, D., and Vollgraf, R. (August 2018) <title rend="quotes">Contextual string embeddings for sequence labeling</title>, in <title rend="italic">Proceedings of the 27th International Conference on Computational Linguistics</title>, pp. 1638–1649.</bibl>
            
            <bibl xml:id="blouin2021" label="Blouin et al. 2021">Blouin et al. (2021) <title rend="quotes">Transferring modern named entity recognition to the historical domain: How to take the step</title>?, in <title rend="italic">Workshop on Natural Language Processing for Digital Humanities</title> (NLP4DH).</bibl>
            
            <bibl xml:id="bojanowski2017" label="Bojanowski et al. 2017">Bojanowski, P. et al. (2017) <title rend="quotes">Enriching word vectors with subword information</title>, <title rend="italic">Transactions of the Association for Computational Linguistics</title>, 5, pp. 135–146.</bibl>
            
            <bibl xml:id="deepseek2025" label="DeepSeek-AI 2025">DeepSeek-AI (2025) <title rend="quotes">Deepseek-r1: Incentivizing reasoning capability in LLMs via reinforcement learning</title>, <title rend="italic">arXiv preprint</title> arXiv:2501.12948.</bibl>
            
            <bibl xml:id="devlin2019" label="Devlin et al. 2019">Devlin, J. et al. (2019) <title rend="quotes">Bert: Pre-training of deep bidirectional transformers for language understanding</title>, in <title rend="italic">Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1</title> (Long and Short Papers), pp. 4171–4186.</bibl>
            
            <bibl xml:id="ehrmann2023" label="Ehrmann 2023">Ehrmann, M. et al. (2023) <title rend="quotes">Named entity recognition and classification in historical documents: A survey</title>, <title rend="italic">ACM Computing Surveys</title>, 56(2), pp. 1–47.</bibl>
            
            <bibl xml:id="gonzález2023" label="González-Gallardo et al. 2023">González-Gallardo, C.E. et al. (2023) <title rend="quotes">Yes but.. can chatgpt identify entities in historical documents?</title>, in <title rend="italic">2023 ACM/IEEE Joint Conference on Digital Libraries (JCDL)</title>, pp. 184–189.</bibl>
            
            <bibl xml:id="grover2008" label="Grover et al. 2008">Grover, C. et al. (2008) <title rend="quotes">Named entity recognition for digitised historical texts</title>, in <title rend="italic">Proceedings of the International Conference on Language Resources and Evaluation, LREC 2008</title>, pp. 1343–1346.</bibl>
            
            <bibl xml:id="hiltman2025" label="Hiltmann et al. 2025">Hiltmann, T. et al. (2025) <title rend="quotes">NER4all or context is all you need: Using LLMs for low-effort, high-performance NER on historical texts. A humanities informed approach</title>, arXiv preprint arXiv:2502.04351.</bibl>
            
            <bibl xml:id="hodel2023" label="Hodel et al. 2023">Hodel, T., Prada Ziegler, I., and Schneider, C. (2023) <title rend="quotes">Pre-modern data: Applying language modeling and named entity recognition on criminal records in the city of Bern</title>. Presented at the Digital Humanities 2023. Collaboration as Opportunity (DH2023), Graz, Austria. Available at: <ref target="https://doi.org/10.5281/zenodo.8107616">https://doi.org/10.5281/zenodo.8107616</ref>. Accessed: 4 September 2026.</bibl>
            
            <bibl xml:id="keraghel2024" label="Keraghel et al. 2024">Keraghel, I., Morbieu, S., and Nadif, M. (2024) <title rend="quotes">A survey on recent advances in named entity recognition: a comprehensive survey and comparative study</title>. arXiv preprint arXiv:2401.10825.</bibl>
            
            <bibl xml:id="klie2018" label="Klie et al. 2018">Klie, J. C. et al. (2018) <title rend="quotes">The INCEpTION platform: Machine-assisted and knowledge-oriented interactive annotation</title>, in <title rend="italic">Proceedings of the 27th International Conference on Computational Linguistics: System Demonstrations</title>, pp. 5–9.</bibl>
            
            <bibl xml:id="krug2017" label="Krug et al. 2017">Krug, M. et al. (2017) <title rend="quotes">Description of a corpus of character references in German novels - DROC [Deutsches ROman Corpus]</title>. Available at: <ref target="https://publications.goettingen-research-online.de/bitstream/2/108301/1/dwp-2018-27.pdf">https://publications.goettingen-research-online.de/bitstream/2/108301/1/dwp-2018-27.pdf</ref>. Accessed: 4 September 2026.</bibl>
            
            <bibl xml:id="mikolov2013" label="Mikolov et al. 2013">Mikolov, T. et al. (2013) <title rend="quotes">Efficient estimation of word representations in vector space</title>, arXiv preprint arXiv:1301.3781.</bibl>
            
            <bibl xml:id="openai2023" label="OpenAI 2023">OpenAI (2023) <title rend="quotes">GPT-4 technical report</title>, arXiv preprint arXiv:2303.08774.</bibl>
            
            <bibl xml:id="prada2025" label="Prada Ziegler, Hitz, and Fuchs 2025">Prada Ziegler, I., Hitz, B. and Fuchs, K. (2025) <title rend="quotes">The Basel Land Records Ground Truth: An annotated dataset for information extraction on German-language administrative records</title>, <title rend="italic">Journal of Open Humanities Data</title>, 11(1), 82. Available at: <ref target=" https://doi.org/10.5334/johd.387"> https://doi.org/10.5334/johd.387</ref>. Accessed: 4 September 2026.</bibl>
            
            <bibl xml:id="prada2024" label="Prada Ziegler 2024">Prada Ziegler, I. (2024) <title rend="quotes">Exploration of event extraction techniques in late medieval and early modern administrative records</title>, in <title rend="italic">Proceedings of the Computational Humanities Research Conference 2024</title>, pp. 761–771.</bibl>
            
            <bibl xml:id="prada2024b" label="Prada Ziegler 2024b">Prada Ziegler, I. (2024b) <title rend="quotes">What's in an entity? Exploring nested named entity recognition in the historical land register of Basel (1400-1700)</title>. Presented at DH Benelux 2024, Leuven, Belgium. Available at: <ref target="https://doi.org/10.5281/zenodo.11500543">https://doi.org/10.5281/zenodo.11500543</ref>. Accessed: 4 September 2026.</bibl>
            
            <bibl xml:id="tjong2003" label="Tjong Kim Sang and De Meulder 2003">Tjong Kim Sang, E. F. and De Meulder, F. (2003) <title rend="quotes">Introduction to the CoNLL-2003 shared task: Language-independent named entity recognition</title>, in <title rend="italic">Proceedings of the Seventh Conference on Natural Language Learning at HLT-NAACL 2003</title>, pp. 142–147.</bibl>
            
            <bibl xml:id="schweter2019" label="Schweter and Baiter 2019">Schweter, S. and Baiter, J. (2019) <title rend="quotes">Towards robust named entity recognition for historic German</title>, in <title rend="italic">Proceedings of the 4th Workshop on Representation Learning for NLP (RepL4NLP-2019)</title>, pp. 96–103.</bibl>
            
            <bibl xml:id="schweter2020" label="Schweter and März 2020">Schweter, S. and März, L. (2020) <title rend="quotes">Triple E - Effective ensembling of embeddings and language models for NER of historical German</title>, in <title rend="italic">CLEF (Working Notes)</title>.</bibl>
            
            <bibl xml:id="detoni2022" label="De Toni et al. 2022">De Toni, F. et al. (2022) <title rend="quotes">Entities, dates, and languages: Zero-shot on historical texts with t0</title>, arXiv preprint arXiv:2204.05211.</bibl>
            
            <bibl xml:id="wang2025" label="Wang et al. 2015">Wang, S. et al. (2025) <title rend="quotes">GPT-NER: Named entity recognition via large language models</title>, in <title rend="italic">Findings of the Association for Computational Linguistics: NAACL 2025</title>, pp. 4257–4275.</bibl>        
         </listBibl>
      </back>
   </text>
</TEI>
