\documentstyle[12pt,moretext] {article}

\newcommand\doublesp{\renewcommand\baselinestretch{1.6}\large\normalsize}
\newcommand\singlesp{\renewcommand\baselinestretch{1.05}\large\normalsize}
\newcommand\halfsp{\renewcommand\baselinestretch{1.3}\large\normalsize}
%\renewcommand{\theequation}{\thesection.\arabic{equation}}
%\renewcommand{\thefigure}{\thesection.\arabic{figure}}
%\renewcommand{\thetable}{\thesection.\arabic{table}}
% Let's squeeze some more space out of this puppy:
%
\topmargin -0.35in
%\topmargin 0pt
%\headheight 0pt
%\headsep 0pt
\textheight 9in
\pagestyle{empty}
%\parindent 0pt
\parskip \baselineskip
\oddsidemargin 0in
\evensidemargin 0in
\textwidth 6.5in

\begin {document}
\singlesp
\begin{center}
\begin{Large}
CS 498/698: Assignment 1, Image Formation and Lighting\\
\vspace{0.1in}
Due: 1pm, Tues. Jan. 30, 2001.
\end{Large}
\end{center}
\vspace{0.1in}
\setcounter{equation}{0} \setcounter{section}{0}

\subsection*{1.~Perspective projection [4 marks]}

\subsubsection*{(a) Trucco and Verri, Question 2.3. [2 marks]}

Show that, in the pinhole camera model, three collinear points in 3-D space
are imaged onto three collinear points on the image plane.

Recall the equations of perspective projection:
$$
x=\frac{f}{Z}X; \hspace{0.5in} y=\frac{f}{Z}Y
$$
where ${\bf P}=(X,Y,Z)^T$ is the 3-D point and ${\bf p}=(x,y)^T$ is
the 2-D point, and $f$ is the focal length.

\subsubsection*{(b) Horn, Exercise 2-12. [2 marks]}

Straight lines in the three-dimensional world 
are projected as straight
lines into the two-dimensional image.
The projections of parallel lines intersect at a \emph{vanishing point}
in the image plane.
When does the vanishing point of a family of parallel lines lie at infinity?

In the case of a rectangular object, a great deal of information can be
recovered from lines in the images and their intersections.
The edges of a rectangular solid fall into three sets of parallel lines,
and so give rise to three vanishing points.
In technical drawing one speaks of one-point, two-point, and three-point
perspective.
These terms apply to the cases in which two, one, or none of the three
vanishing points lie at infinity.
What alignment between the edges of the object and the image plane applies
in each case?
(Draw a diagram and/or describe in words.)

\subsection*{2.~Combining images of different exposure [8 marks]}

\noindent{\bf Finding the camera response function.}
Suppose we have $P$ images taken at different exposure times
$\Delta t_j, j=1\ldots P$.
Let $I_{ij}$ be the brightness of pixel $i$ in image $j$.
In each image $i=1\ldots N$ pixels are measured.
The brightness of each pixel is
$$
I_{ij} = f(E_{i}\Delta t_j)
$$
where $I$ is the image brightness, $E$ is the (unknown) scene
brightness, and $f()$ is the (unknown) camera response function that
maps the image brightness onto a discrete set of pixel values.
Note that $f$ is a function of the product of the scene
brightness and the exposure time.
That is, if the exposure time is doubled,
the input to $f$ doubles.
The image brightness, however, is generally a nonlinear function
of scene brightness.

Given a series of measurements, we solve for $f$ as follows:
$$
f^{-1}(I_{ij})=E_i \Delta t_j
$$
$$
\ln{f^{-1}(I_{ij})}=\ln{E_i} + \ln{\Delta t_j}
$$
Letting $g=\ln{f^{-1}}$, gives
$$
g(I_{ij})=\ln{E_i} + \ln{\Delta t_j}
$$
where $i=1\ldots N$ (pixels) and $j=1\ldots P$ (images).

To solve for $g$ we select some subset of pixels (eg., 200 points) in
all frames and minimize the following quadratic function.
$$
\varepsilon^2 = \sum_{i=1\ldots N} \sum_{j=1..P} \left[ w(I_{ij}) \left(
     g(I_{ij}) - \ln{E_i} - \ln{\Delta t_j}\right) \right]^2
    +\lambda \sum_{z=2}^{n-1} \left[ w(z)  g^{\prime\prime}(z)\right]^2
$$
where $g^{\prime\prime}(z) = g(z-1)-2g(z)+g(z+1)$ and
$n$ is the number of grayscale levels in the image.
The first term fits the camera response function, while the
second term acts as a smoothing term that prefers functions with
low curvature.
$\lambda>0$ trades off the effects of these two terms.
A value of $\lambda=100$ was used in our experiments.
$w$ is a weighting function that emphasizes the fit when pixels
are near the midpoint of their range.
$$
w(z) = \left\{ \begin{array}{ll}
                  n-z   & \mbox{if}\ z>n/2\\
                  z     & \mbox{if}\ z\le n/2\\
               \end{array}
       \right.
$$
Finally, we add a constraint
$g(n/2)=0$.
This constrains the function $g$ to be zero at the
midpoint of its domain.
This is necessary since the above equations
only determine $g$ up to a constant factor.

The above function is minimized using the singular value decomposition.
The result is a camera response function $g(z)$, $z=1\ldots n$ and a 
set of brightness values $\ln E_i$, $i=1\ldots N$.

\subsubsection*{(a) Equation counting [2 marks]}

Write the number of \emph{equations}
and \emph{unknowns} for
the above equations in terms of $N$, $P$, and $n$.
Compute these numbers when $N=200$, $P=3$, and $n=256$, the case
in our experiments below.

\noindent{\bf Combining images of different exposure.}
Since we can only determine the image response function $g$ up to a 
scale factor,
we let $\Delta t_j=1$ for a reference image.

For each image $I_j$, we can compute the estimated scene brightness:
$$
\hat{E_j} = \frac{1}{\Delta t_j} f^{-1}(I_j)
          = \frac{1}{\Delta t_j} e^{g({I_j})}
$$

Using all images, we estimate the brightness:
$$
\hat{E} = \sum_{j=1\ldots P} \frac{w(I_j) \hat{E}_j}{Z}
$$
where $Z=\sum_j w_j(I_j)$ and $w$ is the pixel weighting function:
(Note: $w$, $E$, $I_j$, and $Z$ are all functions of pixel position
$(x,y)$.)

To display the composite image, we plot $\log \hat{E}$.
Note that since this accentuates the low intensity information,
it is not directly comparable with the input images $I_j$.

\subsubsection*{(b) Experiments [6 marks]}
In the program, {\tt wyckoff.m}, I use three images, taken with
\emph{exposure values} (EV) as follows: (1.0, 0, -1.0).
A unit change in exposure value corresponds to a doubling
or halving of exposure times.  It may be achieved by either changing
the exposure time, the camera aperature, or both.
Please do the following experiments:
\begin{enumerate}
\item Run the program, print out, and comment on the composite image.
In particular, what details does it capture that are not visible in
the input images?
\item Modify the program to compute $\hat{E_j}$, $i=1\ldots 3$, the
estimated radience from each input image.
To compare the images, use the commands
{\tt imshow(log(E1),[-4 4])},
{\tt imshow(log(E2),[-4 4])},
{\tt imshow(log(E3),[-4 4])}.
This plots each image on a log scale, and sets the ranges ($[-4,4]$)
to be the same.
To what parts of the composite
does each image contribute?
Hint: you can also look at the ``ownership'' images for $I_j$.
\item What is the role of the smoothing term $\lambda$?  To demonstrate
its effect, try fitting the response curve with $\lambda=10$ (low value)
and $\lambda=1000$ (high value).  Print out the plots and comment on
your findings.
\end{enumerate}

\subsection*{3.~Lightness and shading [12 marks]}

\subsubsection*{(a) Lambertian blocks [4 marks]}
Suppose we observe a trihedral junction (ie., a vertex where three
faces meet) of a rectangular block, with known orientation
wrt the camera.
If the block has a Lambertian surface of constant albedo, we can
determine the direction of a single (distant) light source from
the relative brightness of the three faces of the block.

If the orientation of each face in the camera's reference frame
is given by unit normal vectors ${\bf\hat{n}}_1$,
${\bf\hat{n}}_2$, and ${\bf\hat{n}}_3$,
respectively, show how to obtain the lighting direction and albedo
of the surface.

Note: You can ignore the $\cos^4{\theta}$ term in the imaging equation
and that we are using orthographic projection.

\subsubsection*{(b) Derivation of photometric stereo [3 marks]}
\noindent(Adapted from Horn, Exercise 10-16)
Suppose we have three images of a Lambertian surface under three different
light source directions.

The three images $E_1$, $E_2$, and
$E_3$ are given by the following equations,
$$
E_1=\rho({\bf\hat{s}}_1 \cdot {\bf\hat{n}}), \hspace{0.1in}
E_2=\rho({\bf\hat{s}}_2 \cdot {\bf\hat{n}}), \hspace{0.1in}
E_3=\rho({\bf\hat{s}}_3 \cdot {\bf\hat{n}})
$$
where $\rho$ is the surface albedo (may vary with position $(x,y)$),
${\bf\hat{n}}$ is the surface normal (unit vector), and
$$
{\bf\hat{s}}_i = \frac{(-p_i,-q_i,1)^T}{\sqrt{1+p_i^2+q_i^2}}
$$
for $i=1,2,3$ give the directions of the light source.

By subtracting pairs of equations, show that:
$$
\rho{\bf\hat{n}}\cdot (E_1 {\bf\hat{s}}_2 - E_2 {\bf\hat{s}}_1) = 0
\hspace{0.1in} {\rm and} \hspace{0.1in}
\rho{\bf\hat{n}}\cdot (E_2 {\bf\hat{s}}_3 - E_3 {\bf\hat{s}}_2) = 0.
$$

The above equation shows that ${\bf\hat{n}}$ is perpendicular to both
$(E_1 {\bf\hat{s}}_2 - E_2 {\bf\hat{s}}_1)$ and
$(E_2 {\bf\hat{s}}_3 - E_3 {\bf\hat{s}}_2)$.
Show that ${\bf\hat{n}}$ must be parallel to
$$
(E_1 {\bf\hat{s}}_2 - E_2 {\bf\hat{s}}_1) \times
(E_2 {\bf\hat{s}}_3 - E_3 {\bf\hat{s}}_2) =
E_2( E_1({\bf\hat{s}}_2 \times {\bf\hat{s}}_3)
    +E_2({\bf\hat{s}}_3 \times {\bf\hat{s}}_1)
    +E_3({\bf\hat{s}}_1 \times {\bf\hat{s}}_2))
$$

Conclude that
$$
\rho {\bf\hat{n}} =
   k(E_1({\bf\hat{s}}_2 \times {\bf\hat{s}}_3)
    +E_2({\bf\hat{s}}_3 \times {\bf\hat{s}}_1)
    +E_3({\bf\hat{s}}_1 \times {\bf\hat{s}}_2))
$$
for some constant $k$.

By taking the dot product of this relationship with ${\bf\hat{s}}_1$,
and remembering that $E_1=\rho({\bf\hat{s}}_1 \cdot {\bf\hat{n}})$,
show that
$$
\rho({\bf\hat{n}}\cdot{\bf\hat{s}}_1) = k E_1
 \left[{\bf\hat{s}}_1 {\bf\hat{s}}_2 {\bf\hat{s}}_3 \right]
$$
where $\left[{\bf\hat{s}}_1 {\bf\hat{s}}_2 {\bf\hat{s}}_3 \right]$ is
the triple product ${\bf\hat{s}}_1 \cdot ({\bf\hat{s}}_2 \times
{\bf\hat{s}}_3)$.

Finally, show that
$k=\frac{1}{\left[{\bf\hat{s}}_1 {\bf\hat{s}}_2 {\bf\hat{s}}_3\right]}$,
so that
$$
\rho {\bf\hat{n}} =
\frac{(E_1({\bf\hat{s}}_2 \times {\bf\hat{s}}_3)
      +E_2({\bf\hat{s}}_3 \times {\bf\hat{s}}_1)
      +E_3({\bf\hat{s}}_1 \times {\bf\hat{s}}_2))}
     {\left[{\bf\hat{s}}_1 {\bf\hat{s}}_2 {\bf\hat{s}}_3 \right]}
$$

\subsubsection*{(c) Photometric stereo experiment [5 marks]}

I have given you three images of an unknown surface illuminated from
three lightsource directions:
${\bf{s}}_1=[0,0,1]^T,
{\bf{s}}_2=[-1,1,2]^T,
{\bf{s}}_3=[1,0,3]^T$.
Assuming the light source has the same radiance at each position
(ie., $E_1=E_2=E_3$).
Use the expression in part (b) to compute the surface normal and
albedo at each point.
To accomplish this, complete the program {\tt photometric.m},
and show plots of surface normals and
albedo $\rho$.
(Note: Since we don't know $E$, we can only determine $\rho$ only
up to a constant factor.)

\end{document}
