% This file should be replaced with your file with an appendices (headings below are examples only)

% For compilation piecewise (see projekt.tex), it is necessary to uncomment it and change
% \documentclass[../projekt.tex]{subfiles}
% \begin{document}

% Placing of table of contents of the memory media here should be consulted with a supervisor
%\chapter{Contents of the included storage media}

%\chapter{Manual}

%\chapter{Configuration file}

%\chapter{Scheme of RelaxNG configuration file}

%\chapter{Poster}

\chapter{Code segments}

\section{Dictionary utilization}
\label{app:dictionary_utilization}
\begin{minipage}{\linewidth}
\begin{lstlisting}[
language=Python, 
caption={Program segment responsible for the application of substitutions based on bilingual dictionary. First, it tokenizes the source sentence using the OpenNMT tokenizer's Python bindings\protect\footnotemark, then apply the mappings provided by \texttt{vocab}. Each entry of \textit{vocab} dictionary contains a list of translations read from the vocabulary file. Full algorithm that performs the reading of dictionary file and application of substitutions is provided by \texttt{sub\_by\_dict.py} script.},
frame=trBL
]
import pyonmttok
import random

tokenizer = pyonmttok.Tokenizer(mode='aggressive')
with open(src_path, 'r') as src, open(tgt_path, 'w') as tgt:
  sentences = []
    for line in src:
      tokens = tokenizer.tokenize(line)
      for idx, token in enumerate(tokens):
        if token in vocab:
          replaced += 1
          total += 1
        elif token.isalpha():
          total += 1
        if sample:
            token = random.sample(vocab.get(token, [token]), 1)[0]
        else:
            token = vocab.get(token, [token])[0]
      if replaced/float(total) >= 0.1:
        sentences.append(' '.join(tokens))
      else:
        sentences.append(line)
  tgt.write('\n'.join(sentences))
\end{lstlisting}
\end{minipage}
\footnotetext{\href{https://github.com/OpenNMT/Tokenizer/tree/master/bindings/python}{https://github.com/OpenNMT/Tokenizer/tree/master/bindings/python}}

\chapter{Determining the optimal embedding size}
\label{app:find_emb}
Given the fact that a huge parallel corpus (>1M sentences) is needed to train an NMT model and that the model will have only minimal data to use in the case of the first set of experiments described in Section \ref{section:training_small}; it was decided to optimise model performance by finding the optimal embedding size in advance for both pure MT training and MLM+MT training and then use the resulting optimal embedding size in subsequent experiments.

In order to do this, I tested the performance of models that were trained with 128, 256 and 512 embeddings sizes (other parameters remained unchanged for all 3 cases).

\begin{figure}[H]
    \centering
    \subfloat[Pure MT models] {
        \includegraphics[width=0.48\linewidth]{obrazky-figures/find_emb_mt.pdf}
    } 
    \subfloat[Pre-trained MT models]{
        \includegraphics[width=0.48\linewidth]{obrazky-figures/find_emb_mlm+mt.pdf}
    }
    \caption{As it can be seen in graph (a) there is no definite superiority of models with some specific embedding size across language pairs.  However, the case depicted in (b) -- MT models that were first pre-trained with MLM objective -- demonstrates that for both benchmarks and for all language pairs models that utilize embeddings of size 256 show a slight improvement in performance compared to those utilizing other embedding sizes.}
    \label{fig:find_emb}
\end{figure}

\chapter{Training arguments}
\label{app:training_args}
In this appendix, I want to provide the reader with all the information needed to replicate the training experiments that were conducted during my research. Here will be listed 3 execution templates (for TLM/MLM pre-training, for BT training and MT training/fine-tuning) and tables corresponding to each of these templates, which will list the values of arguments to insert instead of corresponding placeholders.

\begin{lstlisting}[
caption={Template for the execution of LM training.},
frame=trBL
]
python XLM/train.py  \
    --lgs en-<tgt> --mlm_steps <MLM steps> --emb_dim <embedding size> \
    --n_layers <number of layers> --n_heads <number of heads> \
    --dropout <dropout> --attention_dropout <attention dropout> \
    --gelu_activation true --batch_size <batch size> \
    --optimizer adam,lr=<learning rate>,weight_decay=<weight decay> \
    --epoch_size <size of training set> --max_epoch 1000 \
    --validation_metrics _valid_mlm_ppl --stopping_criterion _valid_mlm_ppl,10
\end{lstlisting}

\begin{lstlisting}[
caption={Template for the execution of BT training.},
frame=trBL
]
python XLM/train.py \
  --reload_model <model path> --lgs en-<tgt> --bt_steps en-<tgt>-en,<tgt>-en-<tgt> \
  --ae_steps en,<tgt> --lambda_ae 0:1,100000:0.1,300000:0 --encoder_only false \
  --word_dropout 0.1 --word_blank 0.1 --emb_dim <embedding size> \
  --n_layers <number of layers> --n_heads <number of heads> \
  --dropout <dropout> --attention_dropout <attention dropout> \
  --epoch_size <size of training set> --max_epoch 1000 \
  --gelu_activation true --batch_size <batch size> \
  --optimizer adam,lr=<learning rate>,weight_decay=<weight decay> \
  --eval_bleu true --eval_chrf true \
  --stopping_criterion valid_<language pair>_mt_chrf,10
\end{lstlisting}

\begin{lstlisting}[
caption={Template for the execution of MT training.},
frame=trBL
]
python XLM/train.py \
  --reload_model <model path> --lgs en-<tgt> --mt_steps en-<tgt> \
  --encoder_only false --emb_dim <embedding size> --n_layers <number of layers> \
  --n_heads <number of heads> --dropout <dropout> --attention_dropout <attention dropout> \
  --epoch_size <size of training set> --max_epoch 1000 \
  --gelu_activation true --batch_size <batch size> \
  --optimizer adam,lr=<learning rate>,weight_decay=<weight decay> \
  --eval_bleu true --eval_chrf true \
  --stopping_criterion valid_<language pair>_mt_chrf,10
\end{lstlisting}

Hyperparameters enclosed within "<" and ">" that are listed in training execution templates are described in tables each corresponding to some experiment. If some parameter should be excluded, it is denoted as "-". The model path column corresponds to the path to the previously pre-trained model listed in the same group from one of the previous tables.

\begin{table}[H]
    \renewcommand*{\arraystretch}{1.3}
    \centering
    \setlength{\tabcolsep}{2pt}
    \scalebox{0.65}{
    \begin{tabular}{|c|c|c|c|c|c|c|c|c|c|c|c|c|}
        \hline
        \# & Model type & Model Path & MLM steps & E. size & N. of layers & N. of heads & Dropout & A. dropout & B. size & L. rate & W. decay & Codes  \\
        \hline
        \hline
        \multicolumn{13}{|c|}{\textbf{Models trained with a small amount of additional monolingual data.}} \\
        \hline
        1 & \multirow{3}{*}{MLM} & - & \multirow{3}{*}{en,<tgt>} & 128 & \multirow{4}{*}{6} & \multirow{4}{*}{8} & \multirow{4}{*}{0.1} & \multirow{4}{*}{0.1} & \multirow{4}{*}{64} & \multirow{4}{*}{0.0001} & \multirow{4}{*}{0.1} & \multirow{4}{*}{5k} \\
        \cline{1-1}\cline{5-5}
        2 & & - & & 256 & & & & & & & &  \\
        \cline{1-1}\cline{5-5}
        3 & & - & & 512 & & & & & & & &  \\
        \cline{1-2}\cline{4-5}
        4 & TLM & - & en,<tgt>,en-<tgt> & 256 & & & & & & & &  \\
        \hline
        \hline
        \multicolumn{13}{|c|}{\textbf{Models pre-trained on a small amount of non-parallel data}} \\
        \hline
        1 & ACP & - &  en,<tgt> & 256 & 6 & 8 & 0.1 & 0.1 & 64 & 0.0001 & 0.1 & 5k \\
        \hline
        \hline
        \multicolumn{13}{|c|}{\textbf{Models trained with a big amount of additional monolingual data.}} \\
        \hline
        1 & TLM & - & en,<tgt>,en-<tgt> & 512 & 6 & 8 & 0.1 & 0.1 & 64 & 0.0001 & 0.1 & 7k \\
        \hline
        \hline
        \multicolumn{13}{|c|}{\textbf{Fine-tuning the XLM 17 model.}} \\
        \hline
        1 & TLM & XLM17 & en,<tgt>,en-<tgt> & 1280 & 16 & 16 & 0.3 & 0.0 & 16 & 0.00001 & 0 & 175k \\
        \hline
    \end{tabular}}
    \caption{Arguments for LM training}
\end{table}


\begin{table}[H]
    \renewcommand*{\arraystretch}{1.3}
    \centering
    \setlength{\tabcolsep}{2pt}
    \scalebox{0.65}{
    \begin{tabular}{|c|c|c|c|c|c|c|c|c|c|c|c|}
        \hline
        \# & Model type & Model path & E. size &  N. of layers & N. of heads & Dropout & A. dropout & B. size & L. rate & W. decay & Codes \\
        \hline
        \hline
        \multicolumn{12}{|c|}{\textbf{Models pre-trained on a small amount of non-parallel data}} \\
        \hline
        1 & ACP+AT & \#1,\#1 & 256 & 6 & 8 & 0.1 & 0.1 & 64 & 0.0001 & 0.1 & 5k \\
        \hline
    \end{tabular}}
    \caption{Arguments for BT (AT) training}
\end{table}

\begin{table}[H]
    \renewcommand*{\arraystretch}{1.3}
    \centering
    \setlength{\tabcolsep}{2pt}
    \scalebox{0.65}{
    \begin{tabular}{|c|c|c|c|c|c|c|c|c|c|c|c|c}
        \hline
        \# & Model type & Model path &  E. size &  N. of layers & N. of heads & Dropout & A. dropout & B. size & L. rate & W. decay & Codes \\
        \hline
        \hline
        \multicolumn{12}{|c|}{\textbf{Models trained with a small amount of additional monolingual data}} \\
        \hline
        1 & \multirow{3}{*}{Pure MT} & \multirow{3}{*}{-} & 128 & \multirow{7}{*}{6} & \multirow{7}{*}{8} & \multirow{7}{*}{0.1} & \multirow{7}{*}{0.1} & \multirow{7}{*}{64} & \multirow{7}{*}{0.0001} & \multirow{7}{*}{0.1} & \multirow{7}{*}{5k} \\
        \cline{1-1}\cline{4-4}
        2 &                         &         & 256 & & & & & & & & \\
        \cline{1-1}\cline{4-4}
        3 &                         &         & 512 & & & & & & & & \\
        \cline{1-3}\cline{4-4}
        4 & \multirow{4}{*}{MT FT}  & \#1,\#1 & 128 & & & & & & & & \\
        \cline{1-1}\cline{3-3}\cline{4-4}
        5 &                         & \#2,\#2 & 256 & & & & & & & & \\
        \cline{1-1}\cline{3-3}\cline{4-4}
        6 &                         & \#3,\#3 & 512 & & & & & & & & \\
        \cline{1-1}\cline{3-4}
        7 &                         & \#4,\#4 & 256 & & & & & & & & \\
        \hline
        \hline
        \multicolumn{12}{|c|}{\textbf{Models pre-trained on a small amount of non-parallel data}} \\
        \hline
        1 & ACP+AT,MT & \#1,\#1 & 256 & 6 & 8 & 0.1 & 0.1 & 64 & 0.0001 & 0.1 & 5k \\
        \hline
        \hline
        \multicolumn{12}{|c|}{\textbf{Models trained with a big amount of additional monolingual data.}} \\
        \hline
        1 & MT FT & \#1,\#1 & 512 & 6 & 8 & 0.1 & 0.1 & 64 & 0.0001 & 0.1 & 7k \\
        \hline
        \hline
        \multicolumn{12}{|c|}{\textbf{Fine-tuning the XLM 17 model.}} \\
        \hline
        1 & MT FT & XLM17,XLM17 & \multirow{2}{*}{1280} & \multirow{2}{*}{16} & \multirow{2}{*}{16} & \multirow{2}{*}{0.3} & \multirow{2}{*}{0.0} & \multirow{2}{*}{16} & \multirow{2}{*}{0.00001} & \multirow{2}{*}{0} & \multirow{2}{*}{175k} \\
        \cline{1-3}
        2 & 2-stage-FT & \#1,\#1 & & & & & & & & &  \\
        \hline
    \end{tabular}}
    \caption{Arguments for MT training}
\end{table}


\chapter{Models' performance overview}
\label{app:models_performance}

This appendix provides a table overview of the performance of all the models trained for this research on all test sets. Test sets denoted as \textit{dev} and \textit{test} correspond to flores200 dev and devtest sets respectively. BLEU and chrF3 scores are separated with a comma.

\begin{table}[H]
    \renewcommand*{\arraystretch}{1.3}
    \centering
    \setlength{\tabcolsep}{1.9pt}
    \scalebox{0.55}{
    \begin{tabular}{|c|c|ccc|ccc|ccc|ccc|}
        \hline
        \multicolumn{2}{|c|}{\multirow{2}{*}{\textbf{Model}}} & \multicolumn{3}{c|}{\textbf{en-fr}} & \multicolumn{3}{c|}{\textbf{en-vi}} & \multicolumn{3}{c|}{\textbf{en-eu}} & \multicolumn{3}{c|}{\textbf{en-cs}}   \\
        \cline{3-14}
        \multicolumn{2}{|c|}{} & \textbf{dev} & \textbf{test} & \textbf{Tatoeba} & \textbf{dev} & \textbf{test} & \textbf{Tatoeba} & \textbf{dev} & \textbf{test} & \textbf{Tatoeba} & \textbf{dev} & \textbf{test} & \textbf{Tatoeba} \\
        \hline
        \hline
        \multicolumn{14}{|c|}{\textbf{Models trained with a small amount of additional monolingual data.}} \\
        \hline
        \multirow{3}{*}{MT} & Emb128 & 12.58, &	12.87, &	15.24, &	16.00, & 14.95, & 24.48, & 4.06, &	3.84, &	17.9, &	7.48, &	7.12, &	8.5, \\
        & Emb256 & 12.16,39.4 &	11.86,39.42 &	13.25,36.27 &	16.54,36.41 &	16.29,35.56 &	22.94,40.6 &	4.8,37.69 &	4.68,37.14 &	16.24,45.59 &	8.5,34.25 &	8.79,34.62 &	8.64,29.7 \\
        & Emb512 & 11.9,	& 11.63, & 13.45, & 16.65, & 16.25, & 23.16, & 4.96, & 4.58, & 16.3, & 8.17, & 8.5, & 9.24, \\
        \hline
        \multirow{3}{*}{MLM,MT} & Emb128 & 12.74, &	12.58, &	14.52, &	20.85, &	20.22, &	26.74, &	5.91, &	5.63, &	20.19, &	8.84, &	9.15, &	9.64, \\
        & Emb256 & 16.96,43.96 &	16.87,44.39 &	17.87,40.71 &	21.39,41.22 &	21.43,41.06 &	25.5,43.53 &	6.42,40.33 &	6.28,39.93 &	18.45,48.24 &	10.99,37.72 &	10.88,38.06 &	12.43,33.01 \\
        & Emb512 & 16.33, &	16.76, &	16.98, &	20.69, &	19.97, &	25.65, &	6.12, &	5.9, &	17.73, &	10.39, &	10.31, &	11.37, \\
        \hline	
        \multirow{4}{*}{TLM,MT} & Baseline & 18.67,46.15 & 18.19,46.24 & 19.89,42.71 & 22.44,42.18 & 22.39,42.07 & 26.71,44.56 & 6.79,41.17 & 6.9,41.01 & 19.85,49.56 & 11.44,38.7 & 11.17,38.45 & 11.48,32.84 \\
        & BDBsep & 18.96,46.23 &	19.22,46.69 &	20.82,43.66 &	23.54,43.31 &	23.34,43.41 &	27.23,45.31 &	7.55,42.33 &	7.34,42.35 &	18.67,49.00 &	12.00,39.16 &	12.1,39.14 &	12.33,33.82 \\
        & BDBcat & 19.3,47.04 &	18.91,47.03 &	19.53,43.17 &	22.24,42.34 &	22.99,42.83 & 	26.77,44.84 &	7.05,41.92 &	6.81,41.73 &	18.71,49.07 &	11.74,39.39 &	11.73,39.71 &	12.08,34.18 \\
        & Topline & 19.75,47.56 &	19.13,47.4 &	20.44,43.96 & 	23.28,43.36 &	23.6,43.37 &	26.93,45.04 &	7.66,42.58 &	7.33,42.51 &	20.33,50.33 &	11.94,39.28 &	11.31,38.77 &	12.37,34.42 \\
        \hline
        \hline
        \multicolumn{14}{|c|}{\textbf{Models pre-trained on a small amount of non-parallel data.}} \\
        \hline							
        MLM,MT & Baseline & 17.12,44.17 &	17.11,44.49 &	18.44,40.94 &	22.18,42.38 &	22.49,42.28 &	26.43,44.43 &	6.65,40.55 &	6.58,40.45 &	19.08,48.51 &	11.63,38.52 &	11.62,38.57 &	12.49,33.34 \\
        \hline
        ACP,AT & & 2.70,25.77 & 2.50,25.51	& 2.37,19.20 &	3.21,18.10	& 2.95,17.68 &	2.45,14.08 &	1.67,21.34 &	1.63,21.26 &	1.28,17.3 &	3.27,22.05 &	3.18,21.93 &	1.39,13.48 \\
        \hline
        \multirow{2}{*}{ACP,AT,MT} & Clean & 19.90,47.18 &	19.18,47.20 &	19.69,42.17 &	22.67,42.4 &	22.32,42.21 &	25.79,43.64 &	7.42,41.80 &	6.99,41.89 &	18.34,48.30 &	11.6,38.60 & 11.08,38.83 & 11.73,32.88 \\
        & Noisy & 7.36,33.25	& 7.25,33.46 &	6.34,26.39 &	12.12,32.12 &	12.08,31.81 &	11.04,29.13 &	4.25,33.41 &	4.08,33.28 &	10.33,35.03 &	3.80,25.49 &	3.80,25.42 &	3.50,17.58 \\
        \hline
        \hline
        \multicolumn{14}{|c|}{\textbf{Models trained with a big amount of additional monolingual data.}} \\
        \hline				
        \multirow{4}{*}{TLM,MT} & Baseline & 18.66,46.25 &	19.49,47.31 &	20.23,44.43 &	22.94,43.06	& 23.15,43.02 &	26.99,45.49 & & & & & & \\				
        & Topline & 20.46,48.32 & 	20.87,48.90 &	23.34,46.48 &	24.96,44.82 &	25.07,44.85 &	28.72,46.85 & & & & & & \\
        & TLMsep & 18.97,46.34 &	18.92,46.74 &	21.76,44.59 &	24.77,44.62 &	24.58,44.55 &	28.13,46.13  & & & & & & \\
        & TLMcat & 20.07,47.81 &	20.48,48.21 &	21.6,45.65 &	23.53,43.66 &	23.33,43.23 &	27.35,45.57 & & & & & & \\
        \hline
        \hline
        \multicolumn{14}{|c|}{\textbf{Fine-tuning the XLM 17 model.}} \\
        \hline
        MT & Baseline & 24.85,52.14 &	24.85,53.19 &	28.58,51.75 &	28.56,48.41 &	28.76,48.33 &	31.85,49.69 &	9.73,47.70 &	9.83,48.27 &	22.48,53.55 &	14.64,42.53 &	14.32,42.51 &	18.04,41.19 \\
        \hline
        TLM,MT & TLMsep & 24.05,51.73 &	25.08,52.56 &	28.85,52.11 &	28.63,48.37 &	28.83,48.45 &	31.6,49.85 &	10.00,47.48 &	9.50,47.69 &	22.72,53.92 &	14.30,42.16 &	14.57,42.69 &	17.98,41.96 \\
        \hline
    \end{tabular}}
\end{table}


% For compilation piecewise (see projekt.tex), it is necessary to uncomment it
% \end{document}
