EGamma_ElectronTrackingValidation/docs/presentations/2018_10_24/main.tex

\documentclass[english,aspectratio=43,8pt]{beamer}
\usepackage{graphicx}
\usepackage{amssymb}
\usepackage{booktabs}
\usepackage{siunitx}
\usepackage{subcaption}
\usepackage{marvosym}
\usepackage{verbatim}
\usepackage[normalem]{ulem}  % Needed for /sout

\newcommand{\pb}{\si{\pico\barn}}%
\newcommand{\fb}{\si{\femto\barn}}%
\newcommand{\invfb}{\si{\per\femto\barn}}
\newcommand{\GeV}{\si{\giga\electronvolt}}

\hypersetup{colorlinks=true,urlcolor=blue}
\usetheme[]{bjeldbak}


%%% For wider frames
\newcommand\Wider[2][3em]{%
\makebox[\linewidth][c]{%
  \begin{minipage}{\dimexpr\textwidth+#1\relax}
  \raggedright#2
  \end{minipage}%
  }%
}
%%% For wider frames end

\newcommand{\backupbegin}{%
   \newcounter{finalframe}
   \setcounter{finalframe}{\value{framenumber}}
}
\newcommand{\backupend}{%
   \setcounter{framenumber}{\value{finalframe}}
}

\newcommand\blfootnote[1]{%
  \begingroup
  \renewcommand\thefootnote{}\footnote{#1}%
  \addtocounter{footnote}{-1}%
  \endgroup
}

\begin{document}

\title[$e$ Seeding Validation]{Offline Electron Seeding Validation \-- Update}
\author[C. Fangmeier]{\textbf{Caleb Fangmeier} \\ Ilya Kravchenko,  Greg Snow}
\institute[UNL]{University of Nebraska \-- Lincoln}
\date{EGM General Meeting | November 19, 2018}

\titlegraphic{%
\begin{figure}
  \includegraphics[width=1in]{CMSlogo.png}\hspace{0.75in}\includegraphics[width=1in]{nebraska-n.png}
\end{figure}
}

\begin{frame}[plain]
  \titlepage%
\end{frame}

\begin{frame}{Introduction}
  \begin{itemize}
    \item Our goal is to study \textbf{seeding} for the \textbf{offline} GSF tracking with the \textbf{Phase I pixel detector}.
    \item Specifically, we want to optimize the new pixel-matching scheme from HLT for use in off-line reconstruction.
    \item Previous presentation\footnotemark showed efficiency/purity/fake-rate for proposed offline electron seeding working points.
    \item This Talk:
      \begin{itemize}
        \item Explain ``Hit Skipping'' and demonstrate effects on seeding performance
        \item Examine effects of adding pileup on seeding performance.
      \end{itemize}
  \end{itemize}
  \footnotetext[1]{\tiny \url{https://indico.cern.ch/event/697084/#2-update-on-offline-electron-s}}
\end{frame}


\begin{frame}{N-Hit Electron Seeding}
  \begin{columns}
  \begin{column}{0.5\textwidth}
    {\small
    \begin{enumerate}
      \item Using the beam spot, the SC position, and SC energy, propagate a path through the pixels.
      \item Require the first hit to be within a $\delta\phi$ and $\delta z$ window. ($\delta\phi$ and $\delta R$ for FPIX)
      \item $\delta z$ window for first hit is huge as SC and beam spot positions give very little information about $z$.
      \item Forget the SC position, and propagate a new track based on the vertex and first hit positions, and the SC energy.
      \item Progress one-by-one through the remaining hits in the seed and require each one fit within a specified window around the track.
      \item Quit when all hits are matched, or a hit falls outside the window. \textbf{No skipping is allowed.}
    \end{enumerate}
    }
  \end{column}
  \begin{column}{0.5\textwidth}
    \begin{figure}
      \includegraphics[width=0.9\textwidth]{../common/diagrams/seeding_step2.png}
    \end{figure}
    \begin{figure}
      \includegraphics[width=0.9\textwidth]{../common/diagrams/seeding_step3.png}
    \end{figure}
  \end{column}
  \end{columns}
\end{frame}

\begin{frame}{Hit-Skipping}
  \begin{itemize}
    \item Normally, general tracking seeds are made with an iterative procedure
    \item Each iteration masks hits from use in subsequent steps
    \item Reduces combinatorics for CKF tracking
  % \end{itemize}
  \begin{figure}
    \includegraphics[width=0.7\textwidth]{figures/seeding_steps.png}
  \end{figure}
  % \begin{itemize}
    \item Each iteration works on a specific set of combinations of layers. eg. \texttt{initialStep} uses:
  \end{itemize}
  \begin{columns}[t]
    \begin{column}{0.5\textwidth}
      \texttt{BPix1+BPix2+BPix3+BPix4} \\
      \texttt{BPix1+BPix2+BPix3+FPix1\_pos} \\
      \texttt{BPix1+BPix2+BPix3+FPix1\_neg} \\
      \texttt{BPix1+BPix2+FPix1\_pos+FPix2\_pos}
    \end{column}
    \begin{column}{0.5\textwidth}
      \texttt{BPix1+BPix2+FPix1\_neg+FPix2\_neg} \\
      \texttt{BPix1+FPix1\_pos+FPix2\_pos+FPix3\_pos} \\
      \texttt{BPix1+FPix1\_neg+FPix2\_neg+FPix3\_neg} \\
    \end{column}
  \end{columns}
\end{frame}

\begin{frame}{Hit-Skipping}
  \begin{columns}[t]
    \begin{column}{0.6\textwidth}
      \begin{itemize}
        \item When \texttt{NHitElectronSeedProducer} was implemented for HLT, hit skipping was not added.
        \item Consider an example configuration where we are generating first quadruplet, then triplet, and then finally doublet seeds, masking hits along the way.
        \item If we require at least 3 matched hits, the old method \emph{with} hit skipping would create a seed of hits \texttt{BPIX1, BPIX2, BPIX4}.
        \item But new method \emph{without} hit skipping wouldn't make any seed from these hits.
        \item The ``hack'' is to create seeds using only steps \texttt{tripletElectronSeeds}, and \texttt{pixelPairElectronSeeds} with \textbf{no masking}.
        \item Adding skipping and removing the hack would reduce cpu time from redundant seeds.
      \end{itemize}
    \end{column}
  \begin{column}{0.4\textwidth}
    \begin{figure}
      \includegraphics[width=\textwidth]{figures/seeds_available.png}
    \end{figure}
  \end{column}
  \end{columns}
  % \begin{figure}
  %   \includegraphics[width=\textwidth]{figures/hacked_seeding_steps.png}
  % \end{figure}
\end{frame}

\begin{frame}{Hit-Skipping - Performance}
  \begin{columns}[t]
    \begin{column}{0.5\textwidth}
      \begin{itemize}
        \item Enabling hit skipping and removing hack reduces number of seeds by 36\% to 51\%.
        \item 3-5x fewer seeds with respect to old seeding
        \item Efficiency reduced by between 4\% and 6\% to align more with old seeding performance.
        \item Purity improved by about 1\%.
        \item (table in backup)
      \end{itemize}
    \end{column}
    \begin{column}{0.5\textwidth}
      \begin{figure}
        \includegraphics[width=\textwidth]{figures/tracking_roc_curve_dR.png}
      \end{figure}
    \end{column}
  \end{columns}


{\footnotesize
\begin{table}[]
\begin{tabular}{l|cccc}
\hline
Process    & Seeding Method          & $<N_{seeds}>$(no-skip) & $<N_{seeds}>$(with-skip) & Percent Reduction \\ \hline
$t\bar{t}$ & Old - default settings  & -                      & 12.69                    & -                 \\
           & New - HLT settings      & 4.40                   & 2.56                     & 41\%              \\
           & New - ``wide'' settings & 7.28                   & 4.65                     & 36\%              \\ \hline
Drell-Yan  & Old - default settings  & -                      & 11.40                    & -                 \\
           & New - HLT settings      & 4.70                   & 2.32                     & 51\%              \\
           & New - ``wide'' settings & 5.38                   & 2.65                     & 51\%
\end{tabular}
\end{table}
}

\end{frame}

\begin{frame}{Adding Pileup}
      \begin{itemize}
        \item The simhit-rechit linkage that was previously used in efficiency/purity measurements is not saved in \texttt{GEN-SIM-RAW}.
        \item Therefore, the \texttt{DIGI} step was re-run, but \emph{only for the signal event} because \texttt{GEN-SIM-RAW} does not contain \texttt{SIM} information for pileup events.
        \item However, running this instead of the \texttt{RAW2DIGI} step discarded the previously mixed pileup in the \texttt{RAW}.
        \item So even though there is a \texttt{PileupInfo} collection with reasonable values, there's no actual pileup hits being used for tracking (caused quite some confusion for me).
        \item In the end, abandon simhit-rechit linkage and just use $\Delta R$ matching for efficiency/purity.
        \item Some technical details in backup.
      \end{itemize}
  \begin{columns}[t]
    \begin{column}{0.5\textwidth}
    \end{column}
    \begin{column}{0.5\textwidth}
    \end{column}
  \end{columns}
\end{frame}

\begin{frame}{Adding Pileup - Issues}
      \begin{itemize}
        % \item All previous results are \emph{without} pileup.
        \item Creating kinematic distributions raises some apparent issues with how the new seeding handles pileup.
        \item Next three slides show pt/eta/phi distributions of GSF tracks resulting from ECAL-Driven seeds
      \end{itemize}
      % \begin{figure}
      %   \includegraphics[width=0.32\textwidth]{figures/gsf_track_kinem_dy_skip.png}
      %   \includegraphics[width=0.32\textwidth]{figures/gsf_track_kinem_dy_skip-pileup.png}
      % \end{figure}
\end{frame}

\begin{frame}{Adding Pileup - Issues - No Skipping, No Pileup}
\Wider{
      \begin{figure}
        \includegraphics[width=0.49\textwidth]{figures/gsf_track_kinem_dy_noskip.png}
        \includegraphics[width=0.49\textwidth]{figures/gsf_track_kinem2d_dy_noskip.png}
      \end{figure}
      \begin{itemize}
        \item Looks basically ok, use this as a baseline for comparison.
      \end{itemize}
}
\end{frame}

\begin{frame}{Adding Pileup - Issues - With Skipping, No Pileup}
\Wider{
      \begin{figure}
        \includegraphics[width=0.49\textwidth]{figures/gsf_track_kinem_dy_skip.png}
        \includegraphics[width=0.49\textwidth]{figures/gsf_track_kinem2d_dy_skip.png}
      \end{figure}
      \begin{itemize}
        \item Concerning dip around $\phi=3$ coming from $\eta \in (0.5, 1.5)$.
      \end{itemize}
}
\end{frame}

\begin{frame}{Adding Pileup - Issues - With Skipping, With Pileup}
\Wider{
      \begin{figure}
        \includegraphics[width=0.49\textwidth]{figures/gsf_track_kinem_dy_skip-pileup.png}
        \includegraphics[width=0.49\textwidth]{figures/gsf_track_kinem2d_dy_skip-pileup.png}
      \end{figure}
      \begin{itemize}
        \item Strangely non-flat $\phi$ distribution, manifests differently in both old and new seeding, but rather more pronounced in new.
        \item Features seem to be somewhat localized in $\phi-\eta$, possibly some kind of detector effect?
      \end{itemize}
}
\end{frame}


\begin{frame}{Adding Pileup - More Issues}
\Wider{
  \begin{figure}
    \includegraphics[width=0.49\textwidth]{figures/number_of_good_seeds_dy_skip.png}
    \includegraphics[width=0.49\textwidth]{figures/number_of_good_seeds_dy_skip-pileup.png}
  \end{figure}
  \begin{itemize}
    \item Relative reduction of number of seeds gone. In fact, with pileup there is a 2.3 to 5 times increase in the number of seeds relative to old seeding method.
  \end{itemize}
}
\end{frame}

\begin{frame}{Conclusions \& Outlook}
  \begin{itemize}
    \item Reintroducing hit skipping in new seeding is implemented and has expected results.
    \item Analyzing MC with pileup mixed highlights potential issues with using new seeding for offline reconstruction.
    \item Further investigation is necessary to determine the source of the issues.
    \item Any ideas for checks or fixes are welcome!
  \end{itemize}
  \blfootnote{\tiny Analysis and ploting code is available at \url{https://git.fangmeier.tech/caleb/EGamma\_ElectronTrackingValidation}}
  \blfootnote{\tiny Additional plots are available at \url{http://t3.unl.edu/~cfangmeier/eg/seeding\_studies\_2018\_11\_18\_19/hists.html}}
\end{frame}

\appendix
\backupbegin%

\begin{frame}
  \begin{center}
    {\Huge BACKUP}
  \end{center}
\end{frame}


\begin{frame}{Definitions}
  \begin{itemize}
    \item \textbf{Sim-Track \--} A track from a simulated electron both originating from the luminous region of CMS (beam-spot +- 5$\sigma$) and having $|\eta|<3.0$.
    \item \textbf{ECAL-Driven Seed \--} A seed created via a matching procedure between Super-Clusters and General Tracking Seeds (Either from \texttt{ElectronSeedProducer} or \texttt{ElectronNHitSeedProducer}). Must have $HOE<0.15$.
    \item \textbf{GSF Track \--} A track from GSF-Tracking resulting from an \textbf{ECAL-Driven Seed}
    % \item \textbf{Seeding Efficiency \--} The fraction of \textbf{Sim-Tracks} that have a matching \textbf{ECAL-Driven Seed} (based on simhit-rechit linkage or $\Delta R$ matching)
    \item \textbf{GSF Tracking Efficiency \--} The fraction of \textbf{Sim-Tracks} that have a matching \textbf{GSF Track} (based on $\Delta R$ matching)
    % \item \textbf{ECAL-Driven Seed Purity \--} The fraction of \textbf{ECAL-Driven Seeds} that have a matching \textbf{Sim-Track}
    \item \textbf{GSF Tracking Purity \--} The fraction of \textbf{GSF Tracks} that have a matching \textbf{Sim-Track}
    \item \textbf{GSF Tracking Fake Rate \--} The fraction of nontruth-matched Super-Clusters which result in at least one \textbf{GSF Track}.
  \end{itemize}
\end{frame}

\begin{frame}{Seeding Performance}
  \begin{figure}
    \includegraphics[height=0.945\textheight]{figures/seeding_performance.png}
  \end{figure}
\end{frame}

\begin{frame}{Matching Window Parameters}
\begin{table}[]
\centering
\begin{tabular}{@{}llrrrr@{}}
\toprule
&  & \textbf{narrow} & \textbf{default (HLT)} & \textbf{wide} & \textbf{extra-wide} \\ \midrule
Hit 1 & dPhiMaxHighEt & \textbf{0.025} & \textbf{0.05} & \textbf{0.1} & \textbf{0.15} \\
 & dPhiMaxHighEtThres & 20.0 & 20.0 & 20.0 & 20.0 \\
 & dPhiMaxLowEtGrad & -0.002 & -0.002 & -0.002 & -0.002 \\
 & dRzMaxHighEt & 9999.0 & 9999.0 & 9999.0 & 9999.0 \\
 & dRzMaxHighEtThres & 0.0 & 0.0 & 0.0 & 0.0 \\
 & dRzMaxLowEtGrad & 0.0 & 0.0 & 0.0 & 0.0 \\ \midrule
Hit 2 & dPhiMaxHighEt & \textbf{0.0015} & \textbf{0.003} & \textbf{0.006} & \textbf{0.009} \\
 & dPhiMaxHighEtThres & 0.0 & 0.0 & 0.0 & 0.0 \\
 & dPhiMaxLowEtGrad & 0.0 & 0.0 & 0.0 & 0.0 \\
 & dRzMaxHighEt & \textbf{0.025} & \textbf{0.05} & \textbf{0.1} & \textbf{0.15} \\
 & dRzMaxHighEtThres & 30.0 & 30.0 & 30.0 & 30.0 \\
 & dRzMaxLowEtGrad & -0.002 & -0.002 & -0.002 & -0.002 \\ \midrule
Hit 3+ & dPhiMaxHighEt & \textbf{0.0015} & \textbf{0.003} & \textbf{0.006} & \textbf{0.009} \\
 & dPhiMaxHighEtThres & 0.0 & 0.0 & 0.0 & 0.0 \\
 & dPhiMaxLowEtGrad & 0.0 & 0.0 & 0.0 & 0.0 \\
 & dRzMaxHighEt & \textbf{0.025} & \textbf{0.05} & \textbf{0.1} & \textbf{0.15} \\
 & dRzMaxHighEtThres & 30.0 & 30.0 & 30.0 & 30.0 \\
 & dRzMaxLowEtGrad & -0.002 & -0.002 & -0.002 & -0.002 \\ \bottomrule
\end{tabular}
\end{table}
\centering
\texttt{NHit} Seeding window parameters. Bold designates modified values.
\end{frame}

\begin{frame}{Samples}
  \begin{itemize}
    \item {\tiny /ZToEE\_NNPDF30\_13TeV-powheg\_M\_120\_200/RunIISummer17DRStdmix-NZSFlatPU28to62\_92X\_upgrade2017\_realistic\_v10-v1}

    \item {\tiny /TT\_TuneCUETP8M2T4\_13TeV-powheg-pythia8/RunIISummer17DRStdmix-NZSFlatPU28to62\_92X\_upgrade2017\_realistic\_v10-v2}
  \end{itemize}

\end{frame}


\begin{frame}[fragile]{cmsDriver invocations}
  \begin{columns}[t]
  \begin{column}{0.5\textwidth}
    {\Large \texttt{RAW}$\rightarrow$Step2 (old) }
{\footnotesize
\begin{verbatim}
cmsDriver.py RAW2Step2 \
    --mc \
    --conditions 92X_upgrade2017_realistic_v10 \
    --era Run2_2017 \
    --eventcontent FEVTDEBUG \
    --datatier GEN-SIM-DIGI-RAW \
    --step DIGI:pdigi_valid,L1,DIGI2RAW \
    --geometry DB:Extended \
    --filein file:input.root \
    --fileout file:step2.root \
    --runUnscheduled
\end{verbatim}
}
    {\Large Step2$\rightarrow$TrackingNtuple (old) }
{\footnotesize
\begin{verbatim}
cmsDriver.py Step2ToTrackingNtuple \
    --mc \
    --conditions 92X_upgrade2017_realistic_v10 \
    --era Run2_2017 \
    --eventcontent RECOSIM,MINIAODSIM,DQM \
    --datatier GEN-SIM-RECO,MINIAODSIM,DQMIO \
    --step RAW2DIGI,L1Reco,RECO,EI,PAT,VALIDATION:@standardValidation+@miniAODValidation \
    --filein file:step2.root \
    --fileout file:trackingNtuple.root \
    --customise Validation/RecoTrack/customiseTrackingNtuple.customiseTrackingNtuple \
    --runUnscheduled
\end{verbatim}
}
  \end{column}
  \begin{column}{0.5\textwidth}
    {\large \texttt{RAW}$\rightarrow$TrackingNtuple (new) }
    {\footnotesize
\begin{verbatim}
cmsDriver.py RAW2TrackingNtuple \
    --mc \
    --conditions 92X_upgrade2017_realistic_v10 \
    --era Run2_2017  \
    --eventcontent FEVTDEBUG \
    --datatier GEN-SIM-RECO \
    --step RAW2DIGI,RECO,EI,PAT,VALIDATION \
    --customise Validation/RecoTrack/customiseTrackingNtuple
.customiseTrackingNtuple \
    --filein file:input.root \
    --fileout file:trackingNtuple.root \
    --runUnscheduled
\end{verbatim}
Additionally, hacks to remove hit truth dependencies from \texttt{TrackingNtuple}
}

  \end{column}
  \end{columns}

\end{frame}

\backupend%

\end{document}