diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..6880808 --- /dev/null +++ b/Makefile @@ -0,0 +1,11 @@ +.PHONY: all clean + +PNG_FILES = $(patsubst %.dot,%.png,$(wildcard *.dot)) + +all: $(PNG_FILES) + +%.png: %.dot + dot -Tpng $< -o $@ + +clean: + rm -f *.png diff --git a/analyze-tests.dot b/analyze-tests.dot new file mode 100644 index 0000000..309357f --- /dev/null +++ b/analyze-tests.dot @@ -0,0 +1,4 @@ +digraph analyzetests { + graph [ dpi = 300 ]; + "Transform .sum file into a Python dictionary +} diff --git a/build-steps.dot b/build-steps.dot new file mode 100644 index 0000000..95366ce --- /dev/null +++ b/build-steps.dot @@ -0,0 +1,16 @@ +digraph buildsteps { + graph [ dpi = 300; ] + "Remove previous build directory" -> "Pull from GDB repository"; + "Pull from GDB repository" -> "Copy previous .sum (if not try build)"; + "Copy previous .sum (if not try build)" -> "Configure GDB"; + "Configure GDB" -> "Compile GDB"; + + "Compile GDB" -> "Test GDB (if supported)" [label=" Normal build?"]; + "Compile GDB" -> "Racy test GDB" [label=" Racy test?"]; + + "Test GDB (if supported)" -> "Calculate regressions"; + "Calculate regressions" -> "Save build results"; + + "Racy test GDB" -> "Analyze racy tests"; + "Analyze racy tests" -> "Save racy test information"; +} diff --git a/build-steps.png b/build-steps.png new file mode 100644 index 0000000..b268a44 Binary files /dev/null and b/build-steps.png differ diff --git a/gdb-bof-cauldron-2019.org b/gdb-bof-cauldron-2019.org index 1fa34b0..83928db 100644 --- a/gdb-bof-cauldron-2019.org +++ b/gdb-bof-cauldron-2019.org @@ -10,6 +10,22 @@ - https://creativecommons.org/licenses/by/4.0/ +* Nomenclature + +- *Worker*: The node that performs the “build”. Usually one per + physical machine/VM. For example, =fedora-x86_64-1= or + =ubuntu-aarch64=. + +- *Factory*: A recipe of how to perform a build. + +- *Builder*: An instance of a factory. For example, + =Fedora-x86_64-m64= or + =Ubuntu-Aarch64-native-extended-gdbserver-m64=. + +- *Scheduler*: Dispatches jobs to a set of builders. Can be triggered + by specific events like a commit in a repository, a /try build/ + request or like a cronjob. + * How was it? - GDB Buildbot started in 2015 as a personal project. @@ -54,6 +70,8 @@ - Mark Wielaard: *1* machine (Fedora =s390x=) + - Kamil Rytarowski: *1* machine (NetBSD =amd64=) + #+BEAMER: \pause - Test results are stored directly on-disk, and “garbage-collected” @@ -61,7 +79,64 @@ * How does it work? -- +[[file:submit-patch.png]] + +* How does it work? @@latex:$^2$@@ + +#+ATTR_LATEX: :width 0.5\textwidth +[[file:build-steps.png]] + +* Racy tests handling (or an attempt to) + +- We keep a list of racy tests (detected weekly through the racy + build analysis). + +#+BEAMER: \pause + +- When a racy build finishes, we include the racy tests in the =xfail= + file for that builder. + +#+BEAMER: \pause + +- We then ignore them when doing normal test builds. However... + whac-a-mole. + +* Test analysis (a.k.a. finding regressions) + +- Transform the current =.sum= file into a Python dict: + + - ~{ 'gdb.base/test1.exp: name1' : 'PASS', 'gdb.base/test1.exp: name2' : 'FAIL', ...}~ + +#+BEAMER: \pause + +- Do the same for the previous =.sum= file. + +#+BEAMER: \pause + +- Iterate over the current =.sum= file's dictionary and do: + + #+BEAMER: \pause + + - If the current key is =XFAIL='ed (i.e., a racy test), ignore it. + + #+BEAMER: \pause + + - If the current key exists in the new dict: + + - If it has the same value, good (not a regression). + + - If it changed from =PASS= to =FAIL=, bad. Report as a + regression. + + - If it changed from =FAIL= to =PASS=, good. Update the baseline. + + #+BEAMER: \pause + + - If the current key doesn't exist in the new dict: + + - If it's a =PASS=, good. Update the baseline. + + - If it's a =FAIL=, bad. Report as a new failure. * Notifications @@ -93,8 +168,9 @@ #+BEAMER: \pause - Better way to store and retrieve test results (current way is - “enough” for what we need, but it can certainly be improved). + “enough” for what we need, but it can certainly be improved). See + Serhei's work and Keith's work. #+BEAMER: \pause -- +- =make -jN=, racy tests and =gdb.threads=. diff --git a/gdb-bof-cauldron-2019.pdf b/gdb-bof-cauldron-2019.pdf index 37a4d39..0758c51 100644 Binary files a/gdb-bof-cauldron-2019.pdf and b/gdb-bof-cauldron-2019.pdf differ diff --git a/gdb-bof-cauldron-2019.tex b/gdb-bof-cauldron-2019.tex index b5ca514..1cbd848 100644 --- a/gdb-bof-cauldron-2019.tex +++ b/gdb-bof-cauldron-2019.tex @@ -1,4 +1,4 @@ -% Created 2019-09-12 Thu 13:03 +% Created 2019-09-14 Sat 15:04 % Intended LaTeX compiler: pdflatex \documentclass[presentation]{beamer} \usepackage[utf8]{inputenc} @@ -31,7 +31,7 @@ \maketitle -\begin{frame}[label={sec:orgbad1c50}]{License} +\begin{frame}[label={sec:org18d2f5e}]{License} \begin{itemize} \item License: \alert{Creative Commons Attribution 4.0 International License (CC-BY-4.0)} @@ -39,7 +39,25 @@ \end{itemize} \end{frame} -\begin{frame}[fragile,label={sec:org4538604}]{How was it?} +\begin{frame}[fragile,label={sec:org4f17476}]{Nomenclature} + \begin{itemize} +\item \alert{Worker}: The node that performs the “build”. Usually one per +physical machine/VM. For example, \texttt{fedora-x86\_64-1} or +\texttt{ubuntu-aarch64}. + +\item \alert{Factory}: A recipe of how to perform a build. + +\item \alert{Builder}: An instance of a factory. For example, +\texttt{Fedora-x86\_64-m64} or +\texttt{Ubuntu-Aarch64-native-extended-gdbserver-m64}. + +\item \alert{Scheduler}: Dispatches jobs to a set of builders. Can be triggered +by specific events like a commit in a repository, a \emph{try build} +request or like a cronjob. +\end{itemize} +\end{frame} + +\begin{frame}[fragile,label={sec:org85a4532}]{How was it?} \begin{itemize} \item GDB Buildbot started in 2015 as a personal project. \end{itemize} @@ -59,7 +77,7 @@ proved too inefficient over time\ldots{} \end{itemize} \end{frame} -\begin{frame}[fragile,label={sec:org9dadf47}]{And now?} +\begin{frame}[fragile,label={sec:org8476c6c}]{And now?} \begin{itemize} \item The master runs in a dedicated VM at \alert{OSCI} (\textbf{O}pen \textbf{S}ource \textbf{C}ommunity @@ -93,6 +111,8 @@ Debian Jessie \texttt{s390x}) \item Edjunior Machado: \alert{1} machine (CentOS 7 \texttt{PPC64LE}) \item Mark Wielaard: \alert{1} machine (Fedora \texttt{s390x}) + +\item Kamil Rytarowski: \alert{1} machine (NetBSD \texttt{amd64}) \end{itemize} \end{itemize} @@ -104,13 +124,95 @@ every week (tests older than 4 months are deleted). \end{itemize} \end{frame} -\begin{frame}[label={sec:orgd76b986}]{How does it work?} +\begin{frame}[label={sec:orgc166782}]{How does it work?} +\begin{center} +\includegraphics[width=.9\linewidth]{submit-patch.png} +\end{center} +\end{frame} + +\begin{frame}[label={sec:orgde5b8ca}]{How does it work? $^2$} +\begin{center} +\includegraphics[width=0.5\textwidth]{build-steps.png} +\end{center} +\end{frame} + +\begin{frame}[fragile,label={sec:org7f13e82}]{Racy tests handling (or an attempt to)} + \begin{itemize} +\item We keep a list of racy tests (detected weekly through the racy +build analysis). +\end{itemize} + +\pause + \begin{itemize} -\item +\item When a racy build finishes, we include the racy tests in the \texttt{xfail} +file for that builder. +\end{itemize} + +\pause + +\begin{itemize} +\item We then ignore them when doing normal test builds. However\ldots{} +whac-a-mole. \end{itemize} \end{frame} -\begin{frame}[label={sec:orge36c4d4}]{Notifications} +\begin{frame}[fragile,label={sec:org05ca5bf}]{Test analysis (a.k.a. finding regressions)} + \begin{itemize} +\item Transform the current \texttt{.sum} file into a Python dict: + +\begin{itemize} +\item \texttt{\{ 'gdb.base/test1.exp: name1' : 'PASS', 'gdb.base/test1.exp: name2' : 'FAIL', ...\}} +\end{itemize} +\end{itemize} + +\pause + +\begin{itemize} +\item Do the same for the previous \texttt{.sum} file. +\end{itemize} + +\pause + +\begin{itemize} +\item Iterate over the current \texttt{.sum} file's dictionary and do: + +\pause + +\begin{itemize} +\item If the current key is \texttt{XFAIL}'ed (i.e., a racy test), ignore it. +\end{itemize} + +\pause + +\begin{itemize} +\item If the current key exists in the new dict: + +\begin{itemize} +\item If it has the same value, good (not a regression). + +\item If it changed from \texttt{PASS} to \texttt{FAIL}, bad. Report as a +regression. + +\item If it changed from \texttt{FAIL} to \texttt{PASS}, good. Update the baseline. +\end{itemize} +\end{itemize} + +\pause + +\begin{itemize} +\item If the current key doesn't exist in the new dict: + +\begin{itemize} +\item If it's a \texttt{PASS}, good. Update the baseline. + +\item If it's a \texttt{FAIL}, bad. Report as a new failure. +\end{itemize} +\end{itemize} +\end{itemize} +\end{frame} + +\begin{frame}[label={sec:org0dac85f}]{Notifications} \begin{itemize} \item To \emph{gdb-testers}: whenever we detect a possible regression in an upstream commit. @@ -136,8 +238,8 @@ notifications are not (just look at \emph{gdb-testers}). \end{itemize} \end{frame} -\begin{frame}[label={sec:orgcf5098b}]{Problems and challenges} -\begin{itemize} +\begin{frame}[fragile,label={sec:org903b48e}]{Problems and challenges} + \begin{itemize} \item Racy testcases. Perhaps the most difficult/persistent problem? \end{itemize} @@ -152,13 +254,14 @@ compare test results and find regressions. \begin{itemize} \item Better way to store and retrieve test results (current way is -“enough” for what we need, but it can certainly be improved). +“enough” for what we need, but it can certainly be improved). See +Serhei's work and Keith's work. \end{itemize} \pause \begin{itemize} -\item +\item \texttt{make -jN}, racy tests and \texttt{gdb.threads}. \end{itemize} \end{frame} \end{document} diff --git a/submit-patch.png b/submit-patch.png new file mode 100644 index 0000000..58d63e2 Binary files /dev/null and b/submit-patch.png differ