Advanced Linear AlgebraMath for LLMs

Advanced Linear Algebra

Advanced Linear Algebra

Exercises Notebook

Converted from exercises.ipynb for web reading.

Eigenvalues and Eigenvectors - Exercises

This notebook contains 10 progressive exercises for 01-Eigenvalues-and-Eigenvectors. Each exercise has a learner workspace followed by a complete reference solution. Use the solution cells after making a serious attempt.

Difficulty grows from direct computation to AI-facing interpretation. Formulas use LaTeX-in-Markdown with $...$ and `

......

`.

Code cell 2

import numpy as np
import matplotlib.pyplot as plt
import matplotlib as mpl

try:
    import seaborn as sns
    sns.set_theme(style="whitegrid", palette="colorblind")
    HAS_SNS = True
except ImportError:
    plt.style.use("seaborn-v0_8-whitegrid")
    HAS_SNS = False

mpl.rcParams.update({
    "figure.figsize":    (10, 6),
    "figure.dpi":         120,
    "font.size":           13,
    "axes.titlesize":      15,
    "axes.labelsize":      13,
    "xtick.labelsize":     11,
    "ytick.labelsize":     11,
    "legend.fontsize":     11,
    "legend.framealpha":   0.85,
    "lines.linewidth":      2.0,
    "axes.spines.top":     False,
    "axes.spines.right":   False,
    "savefig.bbox":       "tight",
    "savefig.dpi":         150,
})
np.random.seed(42)
print("Plot setup complete.")

Code cell 3

import numpy as np
import numpy.linalg as la
import scipy.linalg as sla
from scipy import stats

np.set_printoptions(precision=8, suppress=True)
np.random.seed(42)

COLORS = {
    "primary": "#0077BB",
    "secondary": "#EE7733",
    "tertiary": "#009988",
    "error": "#CC3311",
    "neutral": "#555555",
    "highlight": "#EE3377",
}

def header(title):
    print("\n" + "=" * len(title))
    print(title)
    print("=" * len(title))

def check_true(name, cond):
    ok = bool(cond)
    print(f"{'PASS' if ok else 'FAIL'} - {name}")
    return ok

def check_close(name, got, expected, tol=1e-8):
    ok = np.allclose(got, expected, atol=tol, rtol=tol)
    print(f"{'PASS' if ok else 'FAIL'} - {name}")
    if not ok:
        print("  got     =", got)
        print("  expected=", expected)
    return ok

def softmax(z, axis=-1):
    z = np.asarray(z, dtype=float)
    z = z - np.max(z, axis=axis, keepdims=True)
    e = np.exp(z)
    return e / np.sum(e, axis=axis, keepdims=True)

def gram_schmidt_columns(A, tol=1e-12):
    A = np.asarray(A, dtype=float)
    Q = []
    for j in range(A.shape[1]):
        v = A[:, j].copy()
        for q in Q:
            v -= (q @ v) * q
        n = la.norm(v)
        if n > tol:
            Q.append(v / n)
    return np.column_stack(Q) if Q else np.empty((A.shape[0], 0))

def projection_matrix(A):
    Q = gram_schmidt_columns(A)
    return Q @ Q.T

def numerical_rank(A, tol=1e-10):
    return int(np.sum(la.svd(np.asarray(A, dtype=float), compute_uv=False) > tol))

def stable_rank(A):
    s = la.svd(np.asarray(A, dtype=float), compute_uv=False)
    return float(np.sum(s**2) / (s[0]**2 + 1e-15))

def make_spd(n, seed=0, ridge=0.5):
    rng = np.random.default_rng(seed)
    A = rng.normal(size=(n, n))
    return A.T @ A + ridge * np.eye(n)

print("Chapter 03 helper setup complete.")

Exercise 1: Verify Eigenpairs

Check whether candidate pairs satisfy Av=λvA v = \lambda v and reject the zero vector.

Code cell 5

# Your Solution
# Exercise 1 - learner workspace
# Write your solution here, then run the reference solution below to compare.
print("Learner workspace ready for Exercise 1.")

Code cell 6

# Solution
# Exercise 1 - Verify Eigenpairs
header("Exercise 1: eigenpair verification")
A = np.array([[5.0, 2.0], [2.0, 5.0]])
def is_eigenpair(A, lam, v, tol=1e-8):
    v = np.asarray(v, dtype=float)
    return la.norm(v) > tol and la.norm(A @ v - lam * v) <= tol
cases = [(7.0, [1, 1], True), (3.0, [1, -1], True), (7.0, [2, 2], True), (5.0, [1, 0], False)]
for lam, v, expected in cases:
    got = is_eigenpair(A, lam, np.array(v))
    print(lam, v, got)
    check_true("candidate classification", got == expected)
print("Takeaway: eigenvectors are directions preserved by the map.")

Exercise 2: Characteristic Polynomial

For a 2imes22 imes 2 matrix, compute p(λ)=λ2−tr⁡(A)λ+det⁡(A)p(\lambda)=\lambda^2-\operatorname{tr}(A)\lambda+\det(A).

Code cell 8

# Your Solution
# Exercise 2 - learner workspace
# Write your solution here, then run the reference solution below to compare.
print("Learner workspace ready for Exercise 2.")

Code cell 9

# Solution
# Exercise 2 - Characteristic Polynomial
header("Exercise 2: characteristic polynomial")
A = np.array([[4.0, 2.0], [1.0, 3.0]])
def char_poly_2x2(A):
    return np.array([1.0, -np.trace(A), la.det(A)])
coeffs = char_poly_2x2(A)
roots = np.roots(coeffs)
print("coefficients:", coeffs)
print("roots:", roots)
check_close("roots match eigvals", np.sort(roots), np.sort(la.eigvals(A)))
print("Takeaway: eigenvalues are determinant roots.")

Exercise 3: Diagonalization and Powers

Use A=VΛV−1A=V\Lambda V^{-1} to compute large powers without repeated multiplication.

Code cell 11

# Your Solution
# Exercise 3 - learner workspace
# Write your solution here, then run the reference solution below to compare.
print("Learner workspace ready for Exercise 3.")

Code cell 12

# Solution
# Exercise 3 - Diagonalization and Powers
header("Exercise 3: diagonalization powers")
F = np.array([[1.0, 1.0], [1.0, 0.0]])
def power_via_eig(A, k):
    vals, vecs = la.eig(A)
    Ak = vecs @ np.diag(vals**k) @ la.inv(vecs)
    return np.real_if_close(Ak)
F20 = power_via_eig(F, 20)
print("F^20 =\n", F20)
check_close("F^20 matches numpy", F20, la.matrix_power(F, 20), tol=1e-6)
check_true("Fibonacci entry", abs(F20[0,0] - 10946) < 1e-4)
print("Takeaway: diagonal coordinates turn powers into scalar powers.")

Exercise 4: Spectral Theorem

For symmetric AA, verify A=QΛQTA=Q\Lambda Q^T with orthonormal eigenvectors.

Code cell 14

# Your Solution
# Exercise 4 - learner workspace
# Write your solution here, then run the reference solution below to compare.
print("Learner workspace ready for Exercise 4.")

Code cell 15

# Solution
# Exercise 4 - Spectral Theorem
header("Exercise 4: spectral theorem")
A = np.array([[4.0, 2.0, 0.0], [2.0, 3.0, 1.0], [0.0, 1.0, 2.0]])
vals, Q = la.eigh(A)
Lam = np.diag(vals)
print("eigenvalues:", vals)
check_close("Q^T Q = I", Q.T @ Q, np.eye(3))
check_close("reconstruction", Q @ Lam @ Q.T, A)
check_true("symmetric eigenvalues are real", np.all(np.isreal(vals)))
print("Takeaway: symmetry gives the cleanest possible eigendecomposition.")

Exercise 5: Rayleigh Quotient

Show that RA(x)=xTAx/xTxR_A(x)=x^T A x / x^T x lies between the smallest and largest eigenvalue for symmetric AA.

Code cell 17

# Your Solution
# Exercise 5 - learner workspace
# Write your solution here, then run the reference solution below to compare.
print("Learner workspace ready for Exercise 5.")

Code cell 18

# Solution
# Exercise 5 - Rayleigh Quotient
header("Exercise 5: Rayleigh quotient bounds")
A = np.array([[3.0, 1.0], [1.0, 2.0]])
vals = la.eigvalsh(A)
def rayleigh(A, x):
    return float(x @ A @ x / (x @ x))
for x in [np.array([1.0, 0.0]), np.array([1.0, 1.0]), np.array([-2.0, 1.0])]:
    r = rayleigh(A, x)
    print("x=", x, "R=", r)
    check_true("inside spectral interval", vals[0] - 1e-12 <= r <= vals[-1] + 1e-12)
print("Takeaway: the Rayleigh quotient probes curvature direction by direction.")

Exercise 6: Power Iteration

Implement power iteration and estimate the dominant eigenpair.

Code cell 20

# Your Solution
# Exercise 6 - learner workspace
# Write your solution here, then run the reference solution below to compare.
print("Learner workspace ready for Exercise 6.")

Code cell 21

# Solution
# Exercise 6 - Power Iteration
header("Exercise 6: power iteration")
A = np.array([[2.0, 0.8], [0.8, 4.0]])
def power_iteration(A, steps=30):
    x = np.array([1.0, 1.0])
    x = x / la.norm(x)
    for _ in range(steps):
        x = A @ x
        x = x / la.norm(x)
    lam = float(x @ A @ x)
    return lam, x
lam, v = power_iteration(A)
vals, vecs = la.eigh(A)
print("estimated lambda:", lam)
check_true("dominant eigenvalue accurate", abs(lam - vals[-1]) < 1e-8)
check_close("eigen residual small", A @ v, lam * v, tol=1e-7)
print("Takeaway: spectral gap controls convergence speed.")

Exercise 7: Markov Chain Stationary Vector

Find the stationary distribution as the eigenvector for eigenvalue 11 of a transition matrix.

Code cell 23

# Your Solution
# Exercise 7 - learner workspace
# Write your solution here, then run the reference solution below to compare.
print("Learner workspace ready for Exercise 7.")

Code cell 24

# Solution
# Exercise 7 - Markov Chain Stationary Vector
header("Exercise 7: Markov stationary vector")
P = np.array([[0.8, 0.2, 0.1], [0.1, 0.7, 0.2], [0.1, 0.1, 0.7]])
vals, vecs = la.eig(P)
idx = np.argmin(abs(vals - 1.0))
pi = np.real(vecs[:, idx])
pi = pi / pi.sum()
print("stationary pi:", pi)
check_close("P pi = pi", P @ pi, pi, tol=1e-10)
check_close("probability sum", pi.sum(), 1.0)
print("Takeaway: long-run linear dynamics are spectral objects.")

Exercise 8: Gradient Descent Stability

For f(x)= rac12 x^T Hx, relate the learning-rate limit to λmax⁡(H)\lambda_{\max}(H).

Code cell 26

# Your Solution
# Exercise 8 - learner workspace
# Write your solution here, then run the reference solution below to compare.
print("Learner workspace ready for Exercise 8.")

Code cell 27

# Solution
# Exercise 8 - Gradient Descent Stability
header("Exercise 8: gradient descent stability")
H = np.diag([1.0, 8.0])
lmax = la.eigvalsh(H)[-1]
for eta in [0.1, 0.24, 0.30]:
    M = np.eye(2) - eta * H
    rho = max(abs(la.eigvals(M)))
    print("eta", eta, "spectral radius", rho)
check_true("eta below 2/lmax stable", max(abs(la.eigvals(np.eye(2) - 0.24*H))) < 1)
check_true("eta above 2/lmax unstable", max(abs(la.eigvals(np.eye(2) - 0.30*H))) > 1)
print("Takeaway: curvature eigenvalues set safe optimization step sizes.")

Exercise 9: Attention Matrix Spectrum

Treat a row-stochastic attention matrix as a linear averaging operator and inspect its eigenvalues.

Code cell 29

# Your Solution
# Exercise 9 - learner workspace
# Write your solution here, then run the reference solution below to compare.
print("Learner workspace ready for Exercise 9.")

Code cell 30

# Solution
# Exercise 9 - Attention Matrix Spectrum
header("Exercise 9: attention spectrum")
S = np.array([[2.0, 1.0, 0.0], [0.5, 2.0, 0.5], [0.0, 1.0, 2.0]])
A = softmax(S, axis=1)
vals = la.eigvals(A.T)
print("attention rows sum:", A.sum(axis=1))
print("eigenvalues of A^T:", vals)
check_close("row stochastic", A.sum(axis=1), np.ones(3))
check_true("has eigenvalue 1", np.min(abs(vals - 1.0)) < 1e-10)
print("Takeaway: attention mixing has Markov-chain-like spectral structure.")

Exercise 10: Spectral Radius in Recurrence

Simulate ht=Wht−1h_t=Wh_{t-1} and compare decay/explosion with ho(W) ho(W).

Code cell 32

# Your Solution
# Exercise 10 - learner workspace
# Write your solution here, then run the reference solution below to compare.
print("Learner workspace ready for Exercise 10.")

Code cell 33

# Solution
# Exercise 10 - Spectral Radius in Recurrence
header("Exercise 10: spectral radius recurrence")
Ws = [0.8*np.eye(2), np.array([[1.1, 0.2], [0.0, 0.9]])]
for W in Ws:
    h = np.array([1.0, -1.0])
    norms = []
    for _ in range(20):
        h = W @ h
        norms.append(la.norm(h))
    rho = max(abs(la.eigvals(W)))
    print("rho=", rho, "final norm=", norms[-1])
check_true("stable matrix decays", max(abs(la.eigvals(Ws[0]))) < 1)
check_true("unstable matrix can grow", max(abs(la.eigvals(Ws[1]))) > 1)
print("Takeaway: recurrent stability is governed by spectral radius.")
PreviousNext