tdls::TiledLUppSolverDynamic< T, Config > Struct Template Reference

Runtime-size tiled dense LU factorization with logical partial pivoting and out-of-tile pivot recovery, solving one n x n system per call. More...

#include <solver_dynamic.hpp>

Static Public Member Functions
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr int	num_tiles (const int n) noexcept
	Number of tiles per dimension (last one possibly partial).
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr int	tile_size_at (const int t0, const int n) noexcept
	Extent of the tile starting at row/column t0: tile_size, or less for the last tile.
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	ops_swap_rows (T *TDLS_RESTRICT t, const int k, const int r, const int ke) noexcept
	Row swap k <-> r inside the tile (direct, first ke columns).
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	ops_eliminate_column (T *TDLS_RESTRICT t, const int k, const int re, const int ce) noexcept
	Gaussian elimination of column k inside the diagonal tile.
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	ops_trsm_left_unit (const T *TDLS_RESTRICT lu, T *TDLS_RESTRICT B, const int kd, const int ce) noexcept
	B := L^-1 B, with L the unit lower part of the factored diagonal tile. L is kd x kd, B is kd x ce.
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	ops_trsm_right (const T *TDLS_RESTRICT lu, T *TDLS_RESTRICT B, const int kd, const int re) noexcept
	B := B U^-1, with U the upper part of the factored diagonal tile. U is kd x kd, B is re x kd.
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	ops_gemm_sub (T *TDLS_RESTRICT Ct, const T *TDLS_RESTRICT At, const T *TDLS_RESTRICT Bt, const int re, const int ce, const int kd) noexcept
	Ct -= At*Bt with per-element dot-product accumulation. At is re x kd, Bt is kd x ce, Ct is re x ce.
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	load_tile (const int n, const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT prow, const int col0, T *TDLS_RESTRICT t, const int re, const int ce) noexcept
	Load a re x ce tile through a cached physical-row segment.
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	store_tile (const int n, T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT prow, const int col0, const T *TDLS_RESTRICT t, const int re, const int ce) noexcept
	Store a re x ce tile through a cached physical-row segment.
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	load_tile_piv (const int n, const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, const int row0, const int col0, T *TDLS_RESTRICT t, const int re, const int ce) noexcept
	Load a re x ce tile, reading the permutation inline (one read per row).
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	store_tile_piv (const int n, T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, const int row0, const int col0, const T *TDLS_RESTRICT t, const int re, const int ce) noexcept
	Store a re x ce tile, reading the permutation inline (one read per row).
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	load_tile_piv_lower (const int n, const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, const int row0, const int col0, T *TDLS_RESTRICT t, const int re) noexcept
	Triangular variant of the diagonal-tile load for the forward substitution (the backward one under the column-major layout): only the strict lower triangle of L\U is read.
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	load_tile_piv_upper (const int n, const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, const int row0, const int col0, T *TDLS_RESTRICT t, const int re) noexcept
	Triangular variant of the diagonal-tile load for the backward substitution (the forward one under the column-major layout): only the upper triangle including the diagonal is read.
template<bool oot_diagnostics, bool fuse_rhs>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool	factor_diag_column (const int n, T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride, const int k0, T *TDLS_RESTRICT tile, int &oot_count, const int c, const int ke, T *TDLS_RESTRICT y, const int rhs_stride) noexcept
	One column step of the diagonal-tile factorization: pivot search (in-tile, then out-of-tile recovery), permutation update, row swap or cross-tile pull, column elimination.
template<bool oot_diagnostics, bool fuse_rhs>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool	factor_diag_tile (const int n, T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride, const int k0, T *TDLS_RESTRICT tile, int &oot_count, const int ke, T *TDLS_RESTRICT y=nullptr, const int rhs_stride=1) noexcept
	Factor the ke x ke diagonal tile in registers, with out-of-tile pivot recovery (drives the per-column loop of factor_diag_column).
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	rl_trsm_right_one (const int n, T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT pk, const T *TDLS_RESTRICT tile, const int j0, const int ke, const int je) noexcept
	RL: one row-panel tile update, Akj := L^-1 Akj.
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	rl_trsm_right_push_one (const int n, T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT pk, const T *TDLS_RESTRICT tile, const int j0, const int ke, const int je, const int k0, T *TDLS_RESTRICT y, const int rhs_stride) noexcept
	RL, column-major layout, solve_inplace: rl_trsm_right_one, then the fused U^T forward push. Akj is then the final U(k,j) tile: the solved z_k segment is pushed into the y_j segment while the tile sits in registers.
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	rl_schur_one (const int n, T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT pk, const int *TDLS_RESTRICT pi, const T *TDLS_RESTRICT Aik, const int j0, const int ke, const int ie, const int je) noexcept
	RL: one Schur-complement tile update, Aij -= Aik * Akj, streaming the factored Akj row by row from remote memory.
template<bool fuse_rhs>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	rl_update_row_one (const int n, T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, const int *TDLS_RESTRICT pk, const T *TDLS_RESTRICT tile, const int k, const int i0, const int ke, const int ie, T *TDLS_RESTRICT y=nullptr, const int rhs_stride=1) noexcept
	RL: TRSM down + Schur sweep of one row block below the diagonal, with the optional fused forward-substitution push.
template<bool oot_diagnostics, bool fuse_rhs>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool	rl_step (const int n, T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride, const int k, int &oot_count, T *TDLS_RESTRICT y=nullptr, const int rhs_stride=1) noexcept
	RL: one full factorization step (diagonal tile + trailing updates).
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	ll_correct_tile (const int n, const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, const int row0, const int col0, const int k0, T *TDLS_RESTRICT t, const int re, const int ce) noexcept
	LL: t (re x ce, rows row0.., cols col0..) -= sum over prior tiles bj < k0/tile_size of L(row0.., bj) * U(bj, col0..). All prior tiles are full.
template<bool fuse_rhs>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	ll_update_below_one (const int n, T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, const T *TDLS_RESTRICT tile, const int k0, const int i0, const int ke, const int ie, T *TDLS_RESTRICT y=nullptr, const int rhs_stride=1) noexcept
	LL: correct + TRSM one L-panel tile below the diagonal, with the optional fused forward-substitution push.
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	ll_update_right_one (const int n, T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, const T *TDLS_RESTRICT tile, const int k0, const int j0, const int ke, const int je) noexcept
	LL: correct + TRSM one U-panel tile right of the diagonal.
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	ll_update_right_push_one (const int n, T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, const T *TDLS_RESTRICT tile, const int k0, const int j0, const int ke, const int je, T *TDLS_RESTRICT y, const int rhs_stride) noexcept
	LL, column-major layout, solve_inplace: ll_update_right_one, then the fused U^T forward push. B is then the final U(k,j) tile: the solved z_k segment is pushed into the y_j segment while the tile sits in registers.
template<bool oot_diagnostics, bool fuse_rhs>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool	ll_step (const int n, T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride, const int k, int &oot_count, T *TDLS_RESTRICT y=nullptr, const int rhs_stride=1) noexcept
	LL: one full factorization step (correct + factor the diagonal tile, then its L and U panels).
template<bool check>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr int	physical_factorize_impl (const int n, T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride, const T u, int &weak) noexcept
	Physical factorization: P S = L U in place, L below the diagonal and U above it in the pivoted row order, 1 / pivot on the diagonal; piv[j] the row exchanged with row j at step j. With check, every multiplier of L is also checked against 1 / u.
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool	physical_factorize (const int n, T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride, int &weak) noexcept
	Physical factorization (see physical_factorize_impl).
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr int	physical_factorize_checked (const int n, T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride, const T u) noexcept
	Physical factorization with its multipliers bounded, the check fused into the elimination (the condensation of ArrowheadLUpp under RowInterchange::Physical).
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	physical_permute (const int n, const int w, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride, const bool reverse) noexcept
	Applies the interchanges to w columns of x, in their order (P x) or in reverse order (P^T x).
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	physical_forward (const int n, const int w, const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride) noexcept
	First stage of the physical substitution of w columns: x := L^-1 P x under the row-major layout, x := U^-T x under the column-major one.
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	physical_backward (const int n, const int w, const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride) noexcept
	Second stage of the physical substitution of w columns: x := U^-1 x under the row-major layout, x := P^T L^-T x under the column-major one.
template<int pass_width, int stages>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	physical_multirhs (const int n, const int nrhs, const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride) noexcept
	Physical substitution of nrhs columns in passes of pass_width columns: the first stage, the second, or both.
template<bool oot_diagnostics = true, bool fuse_rhs = false>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool	factorize (int n, T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride, int &oot_count, T *TDLS_RESTRICT y=nullptr, const int rhs_stride=1) noexcept
	Factor A := P*L*U in place.
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool	factorize (int n, T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride) noexcept
	Diagnostics-free factorize overload: no out-of-tile out-parameter at all.
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	fwd_push_one (const int n, const int nrhs, const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride, const int k0, const int m0, const int ke, const int me) noexcept
	Forward push: subtract L(m,k) * x_k from the x_m segment, for nrhs columns at once.
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	fwd_step (const int n, const int nrhs, const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride, const int k) noexcept
	Forward step: unit-lower solve the diagonal tile's segment, then push it into the tiles below.
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	bwd_pull_one (const int n, const int nrhs, const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride, const int k0, const int m0, const int ke, const int me) noexcept
	Backward pull: subtract U(k,m) * x_m from the x_k segment, for nrhs columns at once.
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	bwd_step (const int n, const int nrhs, const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride, const int k) noexcept
	Backward step: pull the trailing contributions, then upper-solve the diagonal tile's segment.
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	bwd_only (const int n, const int nrhs, const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride) noexcept
	Backward pass alone, used by fwd_bwd and by solve_inplace (whose forward pass happens inside the factorization).
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	fwd_bwd (const int n, const int nrhs, const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride) noexcept
	Triangular solves on already-permuted column(s) x. nrhs columns are processed per tile visit, so every L/U tile is loaded once for the whole block instead of once per column (nrhs = 1 = the single-RHS case).
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	permute_inplace (const int n, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride) noexcept
	In-place application of the permutation to one strided column x, by cycle decomposition: the permute half of substitute_inplace, shared with substitute_inplace_multirhs (which applies it to each of its columns).
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	transposed_solve_tile_upper (const T *TDLS_RESTRICT t, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride, const int nrhs, const int k0, const int ke) noexcept
	In-tile forward solve with U^T (the diagonal of the tile holds the pivot reciprocals).
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	transposed_push_one (const T *TDLS_RESTRICT u, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride, const int nrhs, const int k0, const int m0, const int ke, const int me) noexcept
	Forward push with U^T: x_m segment -= U(k,m)^T * x_k.
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	transposed_pull_one (const T *TDLS_RESTRICT l, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride, const int nrhs, const int k0, const int m0, const int ke, const int me) noexcept
	Backward pull with L^T: x_k segment -= L(m,k)^T * x_m.
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	transposed_solve_tile_lower_unit (const T *TDLS_RESTRICT l, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride, const int nrhs, const int k0, const int ke) noexcept
	In-tile backward solve with L^T (unit diagonal).
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	transposed_fwd_step (const int n, const int nrhs, const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride, const int k) noexcept
	Transposed forward step: U^T-solve the diagonal tile's segment, then push it into the following segments.
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	transposed_bwd_step (const int n, const int nrhs, const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride, const int k) noexcept
	Transposed backward step: pull the trailing contributions with L^T, then L^T-solve the diagonal tile's segment.
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	transposed_bwd_only (const int n, const int nrhs, const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride) noexcept
	Transposed backward pass alone, used by transposed_fwd_bwd and by solve_inplace (whose forward pass happens inside the factorization).
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	transposed_fwd_bwd (const int n, const int nrhs, const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride) noexcept
	Transposed triangular sweeps on unpermuted column(s) x: U^T z = x, then L^T w = z, w left in x (x = P^T w is the caller's last step).
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	unpermute_inplace (const int n, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride) noexcept
	x := P^T x in place for one strided column (x[piv[i]] receives x[i]): the inverse rotation of permute_inplace, each cycle carried once from its smallest index, a 64-bit bitmask up to n = 64, a cycle-leader scan beyond. Values are moved, never computed.
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	substitute (int n, const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, const T *TDLS_RESTRICT b, T *TDLS_RESTRICT x, const int rhs_stride) noexcept
	Solve x := U^-1 L^-1 P b from a prior factorize (column-major layout: x := P^T L^-T U^-T b). b and x must not alias (use substitute_inplace for the aliased single-buffer pattern).
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	substitute_canonical (int n, const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, const int col, T *TDLS_RESTRICT x, const int rhs_stride) noexcept
	Solve with b = e_col generated on the fly: the consistent-tangent-operator path.
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	substitute_canonical_multirhs_pass (const int n, const int nrhs, const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, const int col0, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride) noexcept
	One pass of the canonical multi right-hand-side substitution: nrhs canonical columns e_col0 .. e_{col0+nrhs-1} generated in permuted order into x, then one triangular sweep for all nrhs columns together. Under the column-major layout the columns are generated unpermuted, swept through U^T then L^T, then permuted (x = P^T w). Internal engine of substitute_canonical_multirhs.
template<int pass_width = 0>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	substitute_canonical_multirhs (int n, int nrhs, const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, const int col0, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride) noexcept
	nrhs canonical columns e_col0 .. e_{col0+nrhs-1} solved from a prior factorize: the columns of the inverse that a consistent tangent operator needs.
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	substitute_multirhs_pass (const int n, const int nrhs, const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, const T *TDLS_RESTRICT b, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride) noexcept
	One pass of the multi right-hand-side substitution: nrhs columns of b gathered in permuted order into x, then one triangular sweep for all nrhs columns together. Under the column-major layout the columns are copied unpermuted, swept through U^T then L^T, then permuted (x = P^T w). Internal engine of substitute_multirhs.
template<int pass_width = 0>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	substitute_multirhs (int n, int nrhs, const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, const T *TDLS_RESTRICT b, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride) noexcept
	Solve nrhs right-hand-side columns from a prior factorize: X := U^-1 L^-1 P B (column-major layout: X := P^T L^-T U^-T B), column by column. b and x must not alias (use substitute_inplace_multirhs for the aliased single-buffer pattern).
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	substitute_inplace (int n, const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride) noexcept
	Solve in place: x already holds the unpermuted RHS on entry and the solution on exit.
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	substitute_inplace_multirhs_pass (const int n, const int nrhs, const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride) noexcept
	One pass of the in-place multi right-hand-side substitution: nrhs columns of x permuted by the cycle decomposition of permute_inplace, then one triangular sweep for all nrhs columns together. Under the column-major layout the sweeps come first, then the cycle decomposition of unpermute_inplace applies x = P^T w. Internal engine of substitute_inplace_multirhs.
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	forward_inplace_multirhs_pass (const int n, const int nrhs, const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride) noexcept
	Forward stage of one pass of substitute_inplace_multirhs: the permutation and the sweep with L under the row-major layout (x := L^-1 P x), the sweep with U^T under the column-major one (x := U^-T x). Internal engine of forward_inplace_multirhs.
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	backward_inplace_multirhs_pass (const int n, const int nrhs, const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride) noexcept
	Backward stage of one pass of substitute_inplace_multirhs: the sweep with U under the row-major layout (x := U^-1 x), the sweep with L^T then the inverse permutation under the column-major one (x := P^T L^-T x). Internal engine of backward_inplace_multirhs.
template<int pass_width = 0>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	substitute_inplace_multirhs (int n, int nrhs, const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride) noexcept
	nrhs columns solved in place: x holds the nrhs unpermuted right-hand-side columns on entry and the nrhs solutions on exit.
template<int pass_width = 0>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	forward_inplace_multirhs (int n, int nrhs, const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride) noexcept
	First of the two stages of substitute_inplace_multirhs, in passes of pass_width columns: x := L^-1 P x under the row-major layout, x := U^-T x under the column-major one. The caller may read or update the intermediate columns, then backward_inplace_multirhs completes the solve; the two stages in a row match substitute_inplace_multirhs bitwise.
template<int pass_width = 0>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	backward_inplace_multirhs (int n, int nrhs, const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride) noexcept
	Second of the two stages of substitute_inplace_multirhs, in passes of pass_width columns: x := U^-1 x under the row-major layout, x := P^T L^-T x under the column-major one.
template<bool oot_diagnostics = true>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool	solve (int n, T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride, const T *TDLS_RESTRICT b, T *TDLS_RESTRICT x, const int rhs_stride, int &oot_count) noexcept
	factorize + substitute in one call.
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool	solve (int n, T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride, const T *TDLS_RESTRICT b, T *TDLS_RESTRICT x, const int rhs_stride) noexcept
	Diagnostics-free solve overload: no out-of-tile out-parameter at all.
template<int pass_width = 0, bool oot_diagnostics = true>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool	solve_multirhs (int n, int nrhs, T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride, const T *TDLS_RESTRICT b, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride, int &oot_count) noexcept
	factorize + substitute_multirhs in one call.
template<int pass_width = 0>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool	solve_multirhs (int n, int nrhs, T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride, const T *TDLS_RESTRICT b, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride) noexcept
	Diagnostics-free solve_multirhs overload: no out-of-tile out-parameter at all.
template<bool oot_diagnostics = true>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool	solve_inplace (int n, T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT y, const int rhs_stride, int &oot_count) noexcept
	Factorization with the forward substitution folded in.
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool	solve_inplace (int n, T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT y, const int rhs_stride) noexcept
	Diagnostics-free solve_inplace overload: no out-of-tile out-parameter at all.
template<int pass_width = 0, bool oot_diagnostics = true>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool	solve_inplace_multirhs (int n, int nrhs, T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT y, const int rhs_stride, const std::size_t xcol_stride, int &oot_count) noexcept
	factorize + substitute_inplace_multirhs in one call: y holds the nrhs unpermuted right-hand-side columns on entry and the nrhs solutions on exit.
template<int pass_width = 0>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool	solve_inplace_multirhs (int n, int nrhs, T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT y, const int rhs_stride, const std::size_t xcol_stride) noexcept
	Diagnostics-free solve_inplace_multirhs overload: no out-of-tile out-parameter at all.

Static Public Attributes
static constexpr int	tile_size = Config.tile_size
	tile size (int)
static constexpr Schedule	schedule
	elimination schedule (RightLooking or LeftLooking)
static constexpr T	oot_pivot_threshold = Config.oot_pivot_threshold
	Acceptable-pivot threshold of the out-of-tile search, read once from the configuration (see TiledLUppConfig::oot_pivot_threshold).
static constexpr T	relative_pivot_threshold = Config.relative_pivot_threshold
	Relative threshold of the in-tile pivot choice, read once from the configuration (see TiledLUppConfig::relative_pivot_threshold).
static constexpr T	singular_floor = Config.singular_floor
	Singularity floor of the out-of-tile recovery, read once from the configuration (see TiledLUppConfig::singular_floor).
static constexpr bool	transposed = Config.layout == MatrixLayout::ColMajor
	Whether the solve uses the transpose of the factored matrix: S = A^T under the column-major layout (see the file documentation of solver_static.hpp).
static constexpr bool	physical = Config.row_interchange == RowInterchange::Physical
	Whether the rows are interchanged physically, the whole matrix a single tile (see TiledLUppConfig::row_interchange).

Detailed Description

template<typename T, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>
struct tdls::TiledLUppSolverDynamic< T, Config >

Runtime-size tiled dense LU factorization with logical partial pivoting and out-of-tile pivot recovery, solving one n x n system per call.

All entry points are static, host- and device-callable, and take the dimension n plus raw pointers pre-offset by the caller with one runtime stride per array. Entry points:

• factorize: A := P*L*U in place, pivots out (RL or LL)
• substitute: x := U^-1 L^-1 P b (b and x distinct)
• substitute_canonical: idem with b = e_col (tangent-operator columns)
• substitute_inplace: idem with b == x (cycle-decomposition permute)
• solve: factorize + substitute
• solve_inplace: factorize with the forward pass folded in

These are the row-major formulas. Under the column-major layout every substitution computes x := P^T L^-T U^-T b instead (see the layout documentation of the compile-time solver).

Each substitution and solve entry point has a _multirhs twin taking nrhs right-hand-side columns per call (substitute_multirhs, substitute_canonical_multirhs, substitute_inplace_multirhs, solve_multirhs, solve_inplace_multirhs). The columns are solved in passes of pass_width columns (0 = one single pass); every L/U tile is loaded once per pass, and per-column results match nrhs single-column calls bitwise whatever the cutting, under the same contraction of multiply-adds (see above). nrhs is a runtime argument here, as every dimension of this variant (the compile-time solver takes it as a template parameter, under the same name). pass_width is a scheduling knob, not a dimension: it stays a template parameter defaulting to 0, as in the compile-time solver and in the adaptors, so the common case needs no cutting argument at all.

The factored diagonal holds the RECIPROCALS of the U pivots, exactly as in TiledLUppSolverStatic: a factorization produced here must be consumed by the substitution routines of this family.

Template Parameters

T	scalar type (float, double or long double)
Config	compile-time knobs, passed as a constexpr value: tile size (may exceed n), schedule, pivoting thresholds; see TiledLUppConfig (unroll_loops is ignored by this variant)

Member Function Documentation

backward_inplace_multirhs()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int pass_width = 0>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverDynamic< T, Config >::backward_inplace_multirhs ( int n, int nrhs, const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride )

inlinestaticconstexprnoexcept

Second of the two stages of substitute_inplace_multirhs, in passes of pass_width columns: x := U^-1 x under the row-major layout, x := P^T L^-T x under the column-major one.

Template Parameters

pass_width

columns per pass, as in substitute_inplace_multirhs

Parameters

[in]	n	system dimension
[in]	nrhs	total number of right-hand-side columns
[in]	A	factored matrix produced by factorize
[in]	A_stride	element stride of A
[in]	piv	permutation produced by factorize
[in]	piv_stride	element stride of piv
[in,out]	x	the intermediate columns on entry, the nrhs solutions on exit
[in]	rhs_stride	element stride of x
[in]	xcol_stride	element stride between columns of x

backward_inplace_multirhs_pass()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverDynamic< T, Config >::backward_inplace_multirhs_pass ( const int n, const int nrhs, const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride )

inlinestaticconstexprnoexcept

Backward stage of one pass of substitute_inplace_multirhs: the sweep with U under the row-major layout (x := U^-1 x), the sweep with L^T then the inverse permutation under the column-major one (x := P^T L^-T x). Internal engine of backward_inplace_multirhs.

Parameters

[in]	n	system dimension
[in]	nrhs	number of columns of this pass
[in]	A	factored matrix produced by factorize
[in]	A_stride	element stride of A
[in]	piv	permutation produced by factorize
[in]	piv_stride	element stride of piv
[in,out]	x	nrhs columns
[in]	rhs_stride	element stride of x
[in]	xcol_stride	element stride between columns of x

bwd_only()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverDynamic< T, Config >::bwd_only ( const int n, const int nrhs, const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride )

inlinestaticconstexprnoexcept

Backward pass alone, used by fwd_bwd and by solve_inplace (whose forward pass happens inside the factorization).

Parameters

[in]	n	system dimension
[in]	nrhs	number of columns processed together
[in]	A	factored matrix (caller-pre-offset)
[in]	A_stride	element stride of A
[in]	piv	permutation from factorize
[in]	piv_stride	element stride of piv
[in,out]	x	solution column(s)
[in]	rhs_stride	element stride of x
[in]	xcol_stride	element stride between columns of x

bwd_pull_one()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverDynamic< T, Config >::bwd_pull_one ( const int n, const int nrhs, const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride, const int k0, const int m0, const int ke, const int me )

inlinestaticconstexprnoexcept

Backward pull: subtract U(k,m) * x_m from the x_k segment, for nrhs columns at once.

Parameters

[in]	n	system dimension
[in]	nrhs	number of columns processed together
[in]	A	factored matrix (caller-pre-offset)
[in]	A_stride	element stride of A
[in]	piv	permutation from factorize
[in]	piv_stride	element stride of piv
[in,out]	x	solution column(s)
[in]	rhs_stride	element stride of x
[in]	xcol_stride	element stride between columns of x
[in]	k0	first global row of the updated segment
[in]	m0	first global row of the trailing segment
[in]	ke	extent of the updated segment's tile
[in]	me	extent of the trailing segment's tile

bwd_step()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverDynamic< T, Config >::bwd_step ( const int n, const int nrhs, const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride, const int k )

inlinestaticconstexprnoexcept

Backward step: pull the trailing contributions, then upper-solve the diagonal tile's segment.

Parameters

[in]	n	system dimension
[in]	nrhs	number of columns processed together
[in]	A	factored matrix (caller-pre-offset)
[in]	A_stride	element stride of A
[in]	piv	permutation from factorize
[in]	piv_stride	element stride of piv
[in,out]	x	solution column(s)
[in]	rhs_stride	element stride of x
[in]	xcol_stride	element stride between columns of x
[in]	k	step index (k0 = k*tile_size)

factor_diag_column()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<bool oot_diagnostics, bool fuse_rhs>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool tdls::TiledLUppSolverDynamic< T, Config >::factor_diag_column ( const int n, T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride, const int k0, T *TDLS_RESTRICT tile, int & oot_count, const int c, const int ke, T *TDLS_RESTRICT y, const int rhs_stride )

inlinestaticnodiscardconstexprnoexcept

One column step of the diagonal-tile factorization: pivot search (in-tile, then out-of-tile recovery), permutation update, row swap or cross-tile pull, column elimination.

Template Parameters

oot_diagnostics	compile the out-of-tile counter in or out
fuse_rhs	apply the pivot swaps to the fused RHS y (row-major layout; the column-major one swaps nothing)

Parameters

[in]	n	system dimension
[in,out]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A
[in,out]	piv	permutation (logical -> physical row)
[in]	piv_stride	element stride of piv
[in]	k0	first global row/column of the tile
[in,out]	tile	register-resident diagonal tile
[in,out]	oot_count	out-of-tile search counter (oot_diagnostics)
[in]	c	tile column to factor
[in]	ke	extent of the diagonal tile
[in,out]	y	fused right-hand side (fuse_rhs only)
[in]	rhs_stride	element stride of y

Returns

false when the matrix is singular at this column.

factor_diag_tile()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<bool oot_diagnostics, bool fuse_rhs>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool tdls::TiledLUppSolverDynamic< T, Config >::factor_diag_tile ( const int n, T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride, const int k0, T *TDLS_RESTRICT tile, int & oot_count, const int ke, T *TDLS_RESTRICT y = nullptr, const int rhs_stride = 1 )

inlinestaticnodiscardconstexprnoexcept

Factor the ke x ke diagonal tile in registers, with out-of-tile pivot recovery (drives the per-column loop of factor_diag_column).

Template Parameters

oot_diagnostics	compile the out-of-tile counter in or out
fuse_rhs	apply the pivot swaps to the fused RHS y (row-major layout; the column-major one swaps nothing)

Parameters

[in]	n	system dimension
[in,out]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A
[in,out]	piv	permutation (logical -> physical row)
[in]	piv_stride	element stride of piv
[in]	k0	first global row/column of the tile
[in,out]	tile	register-resident diagonal tile, loaded (and, LL: prior-corrected) by the caller
[in,out]	oot_count	out-of-tile search counter (oot_diagnostics)
[in]	ke	extent of the diagonal tile
[in,out]	y	fused right-hand side (fuse_rhs only)
[in]	rhs_stride	element stride of y

Returns

false on a singular matrix.

factorize() [1/2]

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool tdls::TiledLUppSolverDynamic< T, Config >::factorize ( int n, T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride )

inlinestaticnodiscardconstexprnoexcept

Diagnostics-free factorize overload: no out-of-tile out-parameter at all.

Parameters

[in]	n	system dimension (n >= 1)
[in,out]	A	matrix, pre-offset by the caller
[in]	A_stride	element stride of A
[out]	piv	permutation storage (always caller-provided)
[in]	piv_stride	element stride of piv

Returns

false on a singular matrix.

factorize() [2/2]

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<bool oot_diagnostics = true, bool fuse_rhs = false>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool tdls::TiledLUppSolverDynamic< T, Config >::factorize ( int n, T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride, int & oot_count, T *TDLS_RESTRICT y = nullptr, const int rhs_stride = 1 )

inlinestaticnodiscardconstexprnoexcept

Factor A := P*L*U in place.

A becomes L\U (unit lower L below the diagonal, U above including it, the diagonal holding the pivot RECIPROCALS), under logical row permutation piv (piv[i] = physical row holding logical row i). Under physical row interchanges (Config.row_interchange), L\U is stored in the pivoted row order and piv[j] is the row exchanged with row j at step j.

Template Parameters

oot_diagnostics	when false, the out-of-tile diagnostics are compiled out entirely; use the overload without the out-parameter
fuse_rhs	internal hook of solve_inplace: folds the forward substitution of y into the factorization

Parameters

[in]	n	system dimension (n >= 1)
[in,out]	A	matrix, pre-offset by the caller
[in]	A_stride	element stride of A
[out]	piv	permutation storage (always caller-provided)
[in]	piv_stride	element stride of piv
[out]	oot_count	number of columns whose best in-tile pivot fell below oot_pivot_threshold
[in,out]	y	fused right-hand side (fuse_rhs only)
[in]	rhs_stride	element stride of y

Returns

false on a singular matrix.

forward_inplace_multirhs()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int pass_width = 0>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverDynamic< T, Config >::forward_inplace_multirhs ( int n, int nrhs, const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride )

inlinestaticconstexprnoexcept

First of the two stages of substitute_inplace_multirhs, in passes of pass_width columns: x := L^-1 P x under the row-major layout, x := U^-T x under the column-major one. The caller may read or update the intermediate columns, then backward_inplace_multirhs completes the solve; the two stages in a row match substitute_inplace_multirhs bitwise.

Template Parameters

pass_width

columns per pass, as in substitute_inplace_multirhs

Parameters

[in]	n	system dimension
[in]	nrhs	total number of right-hand-side columns
[in]	A	factored matrix produced by factorize
[in]	A_stride	element stride of A
[in]	piv	permutation produced by factorize
[in]	piv_stride	element stride of piv
[in,out]	x	nrhs right-hand-side columns on entry, the intermediate columns on exit
[in]	rhs_stride	element stride of x
[in]	xcol_stride	element stride between columns of x

forward_inplace_multirhs_pass()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverDynamic< T, Config >::forward_inplace_multirhs_pass ( const int n, const int nrhs, const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride )

inlinestaticconstexprnoexcept

Forward stage of one pass of substitute_inplace_multirhs: the permutation and the sweep with L under the row-major layout (x := L^-1 P x), the sweep with U^T under the column-major one (x := U^-T x). Internal engine of forward_inplace_multirhs.

Parameters

[in]	n	system dimension
[in]	nrhs	number of columns of this pass
[in]	A	factored matrix produced by factorize
[in]	A_stride	element stride of A
[in]	piv	permutation produced by factorize
[in]	piv_stride	element stride of piv
[in,out]	x	nrhs columns
[in]	rhs_stride	element stride of x
[in]	xcol_stride	element stride between columns of x

fwd_bwd()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverDynamic< T, Config >::fwd_bwd ( const int n, const int nrhs, const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride )

inlinestaticconstexprnoexcept

Triangular solves on already-permuted column(s) x. nrhs columns are processed per tile visit, so every L/U tile is loaded once for the whole block instead of once per column (nrhs = 1 = the single-RHS case).

Parameters

[in]	n	system dimension
[in]	nrhs	number of columns processed together
[in]	A	factored matrix (caller-pre-offset)
[in]	A_stride	element stride of A
[in]	piv	permutation from factorize
[in]	piv_stride	element stride of piv
[in,out]	x	solution column(s)
[in]	rhs_stride	element stride of x
[in]	xcol_stride	element stride between columns of x

fwd_push_one()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverDynamic< T, Config >::fwd_push_one ( const int n, const int nrhs, const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride, const int k0, const int m0, const int ke, const int me )

inlinestaticconstexprnoexcept

Forward push: subtract L(m,k) * x_k from the x_m segment, for nrhs columns at once.

Parameters

[in]	n	system dimension
[in]	nrhs	number of columns processed together
[in]	A	factored matrix (caller-pre-offset)
[in]	A_stride	element stride of A
[in]	piv	permutation from factorize
[in]	piv_stride	element stride of piv
[in,out]	x	solution column(s)
[in]	rhs_stride	element stride of x
[in]	xcol_stride	element stride between columns of x
[in]	k0	first global row of the solved segment
[in]	m0	first global row of the target segment
[in]	ke	extent of the solved segment's tile
[in]	me	extent of the target segment's tile

fwd_step()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverDynamic< T, Config >::fwd_step ( const int n, const int nrhs, const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride, const int k )

inlinestaticconstexprnoexcept

Forward step: unit-lower solve the diagonal tile's segment, then push it into the tiles below.

Parameters

[in]	n	system dimension
[in]	nrhs	number of columns processed together
[in]	A	factored matrix (caller-pre-offset)
[in]	A_stride	element stride of A
[in]	piv	permutation from factorize
[in]	piv_stride	element stride of piv
[in,out]	x	solution column(s)
[in]	rhs_stride	element stride of x
[in]	xcol_stride	element stride between columns of x
[in]	k	step index (k0 = k*tile_size)

ll_correct_tile()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverDynamic< T, Config >::ll_correct_tile ( const int n, const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, const int row0, const int col0, const int k0, T *TDLS_RESTRICT t, const int re, const int ce )

inlinestaticconstexprnoexcept

LL: t (re x ce, rows row0.., cols col0..) -= sum over prior tiles bj < k0/tile_size of L(row0.., bj) * U(bj, col0..). All prior tiles are full.

Parameters

[in]	n	system dimension
[in]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A
[in]	piv	permutation (logical -> physical row)
[in]	piv_stride	element stride of piv
[in]	row0	first global row of the corrected tile
[in]	col0	first global column of the corrected tile
[in]	k0	first row/column of the current step
[in,out]	t	register tile being corrected
[in]	re	row extent of the corrected tile
[in]	ce	column extent of the corrected tile

ll_step()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<bool oot_diagnostics, bool fuse_rhs>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool tdls::TiledLUppSolverDynamic< T, Config >::ll_step ( const int n, T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride, const int k, int & oot_count, T *TDLS_RESTRICT y = nullptr, const int rhs_stride = 1 )

inlinestaticnodiscardconstexprnoexcept

LL: one full factorization step (correct + factor the diagonal tile, then its L and U panels).

Template Parameters

oot_diagnostics	compile the out-of-tile counter in or out
fuse_rhs	apply the step to the fused RHS y (solve_inplace)

Parameters

[in]	n	system dimension
[in,out]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A
[in,out]	piv	permutation (logical -> physical row)
[in]	piv_stride	element stride of piv
[in]	k	step index (k0 = k*tile_size)
[in,out]	oot_count	out-of-tile search counter (oot_diagnostics)
[in,out]	y	fused right-hand side (fuse_rhs only)
[in]	rhs_stride	element stride of y

Returns

false on a singular matrix.

ll_update_below_one()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<bool fuse_rhs>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverDynamic< T, Config >::ll_update_below_one ( const int n, T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, const T *TDLS_RESTRICT tile, const int k0, const int i0, const int ke, const int ie, T *TDLS_RESTRICT y = nullptr, const int rhs_stride = 1 )

inlinestaticconstexprnoexcept

LL: correct + TRSM one L-panel tile below the diagonal, with the optional fused forward-substitution push.

Template Parameters

fuse_rhs

push the fused RHS y along (solve_inplace)

Parameters

[in]	n	system dimension
[in,out]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A
[in]	piv	permutation (logical -> physical row)
[in]	piv_stride	element stride of piv
[in]	tile	factored diagonal tile
[in]	k0	first global row/column of the step
[in]	i0	first global row of the updated tile
[in]	ke	extent of the diagonal tile
[in]	ie	row extent of the updated tile
[in,out]	y	fused right-hand side (fuse_rhs only)
[in]	rhs_stride	element stride of y

ll_update_right_one()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverDynamic< T, Config >::ll_update_right_one ( const int n, T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, const T *TDLS_RESTRICT tile, const int k0, const int j0, const int ke, const int je )

inlinestaticconstexprnoexcept

LL: correct + TRSM one U-panel tile right of the diagonal.

Parameters

[in]	n	system dimension
[in,out]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A
[in]	piv	permutation (logical -> physical row)
[in]	piv_stride	element stride of piv
[in]	tile	factored diagonal tile
[in]	k0	first global row/column of the step
[in]	j0	first global column of the updated tile
[in]	ke	extent of the diagonal tile
[in]	je	column extent of the updated tile

ll_update_right_push_one()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverDynamic< T, Config >::ll_update_right_push_one ( const int n, T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, const T *TDLS_RESTRICT tile, const int k0, const int j0, const int ke, const int je, T *TDLS_RESTRICT y, const int rhs_stride )

inlinestaticconstexprnoexcept

LL, column-major layout, solve_inplace: ll_update_right_one, then the fused U^T forward push. B is then the final U(k,j) tile: the solved z_k segment is pushed into the y_j segment while the tile sits in registers.

Parameters

[in]	n	system dimension
[in,out]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A
[in]	piv	permutation (logical -> physical row)
[in]	piv_stride	element stride of piv
[in]	tile	factored diagonal tile
[in]	k0	first global row/column of the step
[in]	j0	first global column of the updated tile
[in]	ke	extent of the diagonal tile
[in]	je	column extent of the updated tile
[in,out]	y	fused right-hand side
[in]	rhs_stride	element stride of y

load_tile()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverDynamic< T, Config >::load_tile ( const int n, const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT prow, const int col0, T *TDLS_RESTRICT t, const int re, const int ce )

inlinestaticconstexprnoexcept

Load a re x ce tile through a cached physical-row segment.

Parameters

[in]	n	system dimension
[in]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A
[in]	prow	physical rows of the tile
[in]	col0	first global column of the tile
[out]	t	destination register tile
[in]	re	tile row extent
[in]	ce	tile column extent

load_tile_piv()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverDynamic< T, Config >::load_tile_piv ( const int n, const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, const int row0, const int col0, T *TDLS_RESTRICT t, const int re, const int ce )

inlinestaticconstexprnoexcept

Load a re x ce tile, reading the permutation inline (one read per row).

Parameters

[in]	n	system dimension
[in]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A
[in]	piv	permutation (logical -> physical row)
[in]	piv_stride	element stride of piv
[in]	row0	first global (logical) row of the tile
[in]	col0	first global column of the tile
[out]	t	destination register tile
[in]	re	tile row extent
[in]	ce	tile column extent

load_tile_piv_lower()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverDynamic< T, Config >::load_tile_piv_lower ( const int n, const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, const int row0, const int col0, T *TDLS_RESTRICT t, const int re )

inlinestaticconstexprnoexcept

Triangular variant of the diagonal-tile load for the forward substitution (the backward one under the column-major layout): only the strict lower triangle of L\U is read.

Parameters

[in]	n	system dimension
[in]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A
[in]	piv	permutation (logical -> physical row)
[in]	piv_stride	element stride of piv
[in]	row0	first global (logical) row of the tile
[in]	col0	first global column of the tile
[out]	t	destination register tile
[in]	re	tile extent

load_tile_piv_upper()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverDynamic< T, Config >::load_tile_piv_upper ( const int n, const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, const int row0, const int col0, T *TDLS_RESTRICT t, const int re )

inlinestaticconstexprnoexcept

Triangular variant of the diagonal-tile load for the backward substitution (the forward one under the column-major layout): only the upper triangle including the diagonal is read.

Parameters

[in]	n	system dimension
[in]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A
[in]	piv	permutation (logical -> physical row)
[in]	piv_stride	element stride of piv
[in]	row0	first global (logical) row of the tile
[in]	col0	first global column of the tile
[out]	t	destination register tile
[in]	re	tile extent

num_tiles()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr int tdls::TiledLUppSolverDynamic< T, Config >::num_tiles ( const int n )

inlinestaticconstexprnoexcept

Number of tiles per dimension (last one possibly partial).

Parameters

[in]

n

system dimension

Returns

ceil(n / tile_size)

ops_eliminate_column()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverDynamic< T, Config >::ops_eliminate_column ( T *TDLS_RESTRICT t, const int k, const int re, const int ce )

inlinestaticconstexprnoexcept

Gaussian elimination of column k inside the diagonal tile.

Scales the sub-column by 1/pivot and updates the trailing block. The RECIPROCAL of the pivot is stored at the diagonal slot: every downstream consumer multiplies instead of dividing.

Parameters

[in,out]	t	register tile
[in]	k	column to eliminate
[in]	re	active row extent of the tile
[in]	ce	active column extent of the tile

ops_gemm_sub()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverDynamic< T, Config >::ops_gemm_sub ( T *TDLS_RESTRICT Ct, const T *TDLS_RESTRICT At, const T *TDLS_RESTRICT Bt, const int re, const int ce, const int kd )

inlinestaticconstexprnoexcept

Ct -= At*Bt with per-element dot-product accumulation. At is re x kd, Bt is kd x ce, Ct is re x ce.

Parameters

[in,out]	Ct	accumulator tile
[in]	At	left factor tile
[in]	Bt	right factor tile
[in]	re	row extent of Ct and At
[in]	ce	column extent of Ct and Bt
[in]	kd	inner extent (columns of At, rows of Bt)

ops_swap_rows()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverDynamic< T, Config >::ops_swap_rows ( T *TDLS_RESTRICT t, const int k, const int r, const int ke )

inlinestaticconstexprnoexcept

Row swap k <-> r inside the tile (direct, first ke columns).

Parameters

[in,out]	t	register tile
[in]	k	destination row (elimination column)
[in]	r	source row to swap in
[in]	ke	active extent of the tile

ops_trsm_left_unit()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverDynamic< T, Config >::ops_trsm_left_unit ( const T *TDLS_RESTRICT lu, T *TDLS_RESTRICT B, const int kd, const int ce )

inlinestaticconstexprnoexcept

B := L^-1 B, with L the unit lower part of the factored diagonal tile. L is kd x kd, B is kd x ce.

Parameters

[in]	lu	factored diagonal tile (L\U)
[in,out]	B	updated register tile
[in]	kd	extent of the factored diagonal tile
[in]	ce	column extent of B

ops_trsm_right()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverDynamic< T, Config >::ops_trsm_right ( const T *TDLS_RESTRICT lu, T *TDLS_RESTRICT B, const int kd, const int re )

inlinestaticconstexprnoexcept

B := B U^-1, with U the upper part of the factored diagonal tile. U is kd x kd, B is re x kd.

The diagonal of lu already holds the pivot reciprocals: no divisions here.

Parameters

[in]	lu	factored diagonal tile (L\U, reciprocal diagonal)
[in,out]	B	updated register tile
[in]	kd	extent of the factored diagonal tile
[in]	re	row extent of B

permute_inplace()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverDynamic< T, Config >::permute_inplace ( const int n, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride )

inlinestaticconstexprnoexcept

In-place application of the permutation to one strided column x, by cycle decomposition: the permute half of substitute_inplace, shared with substitute_inplace_multirhs (which applies it to each of its columns).

Each cycle is rotated once from its smallest index. Up to n = 64 a 64-bit bitmask tracks the visited rows; beyond, a cycle-leader scan is used instead (zero storage, an extra integer walk of each orbit). The dispatch is warp-uniform on GPU (n is the same for every lane). Both paths move the same values in the same order.

Parameters

[in]	n	system dimension
[in]	piv	permutation from factorize
[in]	piv_stride	element stride of piv
[in,out]	x	column to permute
[in]	rhs_stride	element stride of x

physical_backward()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverDynamic< T, Config >::physical_backward ( const int n, const int w, const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride )

inlinestaticconstexprnoexcept

Second stage of the physical substitution of w columns: x := U^-1 x under the row-major layout, x := P^T L^-T x under the column-major one.

Parameters

[in]	n	system dimension
[in]	w	number of columns
[in]	A	factors of physical_factorize
[in]	A_stride	element stride of A
[in]	piv	interchanges
[in]	piv_stride	element stride of piv
[in,out]	x	the columns
[in]	rhs_stride	element stride of x
[in]	xcol_stride	element stride between columns of x

physical_factorize()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool tdls::TiledLUppSolverDynamic< T, Config >::physical_factorize ( const int n, T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride, int & weak )

inlinestaticnodiscardconstexprnoexcept

Physical factorization (see physical_factorize_impl).

Parameters

[in]	n	system dimension
[in,out]	A	matrix on entry, its factors on exit
[in]	A_stride	element stride of A
[out]	piv	interchanges
[in]	piv_stride	element stride of piv
[out]	weak	number of columns whose best candidate fell below oot_pivot_threshold

Returns

false on a singular matrix

physical_factorize_checked()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr int tdls::TiledLUppSolverDynamic< T, Config >::physical_factorize_checked ( const int n, T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride, const T u )

inlinestaticnodiscardconstexprnoexcept

Physical factorization with its multipliers bounded, the check fused into the elimination (the condensation of ArrowheadLUpp under RowInterchange::Physical).

Parameters

[in]	n	system dimension
[in,out]	A	matrix on entry, its factors on exit
[in]	A_stride	element stride of A
[out]	piv	interchanges
[in]	piv_stride	element stride of piv
[in]	u	the multipliers must not exceed 1 / u

Returns

0 when every multiplier passes the check, 1 when one fails it, 2 on a singular matrix

physical_factorize_impl()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<bool check>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr int tdls::TiledLUppSolverDynamic< T, Config >::physical_factorize_impl ( const int n, T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride, const T u, int & weak )

inlinestaticnodiscardconstexprnoexcept

Physical factorization: P S = L U in place, L below the diagonal and U above it in the pivoted row order, 1 / pivot on the diagonal; piv[j] the row exchanged with row j at step j. With check, every multiplier of L is also checked against 1 / u.

Template Parameters

check

bound the multipliers by 1 / u

Parameters

[in]	n	system dimension
[in,out]	A	matrix on entry, its factors on exit
[in]	A_stride	element stride of A
[out]	piv	interchanges
[in]	piv_stride	element stride of piv
[in]	u	bound of the check (check only)
[out]	weak	number of columns whose best candidate fell below oot_pivot_threshold (the out-of-tile count of Logical over one tile)

Returns

0 when every multiplier passes the check (always without check), 1 when one fails it, 2 on a singular matrix

physical_forward()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverDynamic< T, Config >::physical_forward ( const int n, const int w, const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride )

inlinestaticconstexprnoexcept

First stage of the physical substitution of w columns: x := L^-1 P x under the row-major layout, x := U^-T x under the column-major one.

Parameters

[in]	n	system dimension
[in]	w	number of columns
[in]	A	factors of physical_factorize
[in]	A_stride	element stride of A
[in]	piv	interchanges
[in]	piv_stride	element stride of piv
[in,out]	x	the columns
[in]	rhs_stride	element stride of x
[in]	xcol_stride	element stride between columns of x

physical_multirhs()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int pass_width, int stages>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverDynamic< T, Config >::physical_multirhs ( const int n, const int nrhs, const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride )

inlinestaticconstexprnoexcept

Physical substitution of nrhs columns in passes of pass_width columns: the first stage, the second, or both.

Template Parameters

pass_width	columns per pass (0: one pass)
stages	1 the first stage, 2 the second, 3 both

Parameters

[in]	n	system dimension
[in]	nrhs	number of columns
[in]	A	factors of physical_factorize
[in]	A_stride	element stride of A
[in]	piv	interchanges
[in]	piv_stride	element stride of piv
[in,out]	x	the columns
[in]	rhs_stride	element stride of x
[in]	xcol_stride	element stride between columns of x

physical_permute()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverDynamic< T, Config >::physical_permute ( const int n, const int w, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride, const bool reverse )

inlinestaticconstexprnoexcept

Applies the interchanges to w columns of x, in their order (P x) or in reverse order (P^T x).

Parameters

[in]	n	system dimension
[in]	w	number of columns
[in]	piv	interchanges
[in]	piv_stride	element stride of piv
[in,out]	x	the columns
[in]	rhs_stride	element stride of x
[in]	xcol_stride	element stride between columns of x
[in]	reverse	P^T instead of P

rl_schur_one()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverDynamic< T, Config >::rl_schur_one ( const int n, T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT pk, const int *TDLS_RESTRICT pi, const T *TDLS_RESTRICT Aik, const int j0, const int ke, const int ie, const int je )

inlinestaticconstexprnoexcept

RL: one Schur-complement tile update, Aij -= Aik * Akj, streaming the factored Akj row by row from remote memory.

Parameters

[in]	n	system dimension
[in,out]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A
[in]	pk	physical rows of the factored block row
[in]	pi	physical rows of the updated block row
[in]	Aik	register L panel of the updated block row
[in]	j0	first global column of the updated tile
[in]	ke	inner extent (current step)
[in]	ie	row extent of the updated tile
[in]	je	column extent of the updated tile

rl_step()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<bool oot_diagnostics, bool fuse_rhs>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool tdls::TiledLUppSolverDynamic< T, Config >::rl_step ( const int n, T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride, const int k, int & oot_count, T *TDLS_RESTRICT y = nullptr, const int rhs_stride = 1 )

inlinestaticnodiscardconstexprnoexcept

RL: one full factorization step (diagonal tile + trailing updates).

Template Parameters

oot_diagnostics	compile the out-of-tile counter in or out
fuse_rhs	apply the step to the fused RHS y (solve_inplace)

Parameters

[in]	n	system dimension
[in,out]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A
[in,out]	piv	permutation (logical -> physical row)
[in]	piv_stride	element stride of piv
[in]	k	step index (k0 = k*tile_size)
[in,out]	oot_count	out-of-tile search counter (oot_diagnostics)
[in,out]	y	fused right-hand side (fuse_rhs only)
[in]	rhs_stride	element stride of y

Returns

false on a singular matrix.

rl_trsm_right_one()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverDynamic< T, Config >::rl_trsm_right_one ( const int n, T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT pk, const T *TDLS_RESTRICT tile, const int j0, const int ke, const int je )

inlinestaticconstexprnoexcept

RL: one row-panel tile update, Akj := L^-1 Akj.

Parameters

[in]	n	system dimension
[in,out]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A
[in]	pk	physical rows of the diagonal block row
[in]	tile	factored diagonal tile
[in]	j0	first global column of the updated tile
[in]	ke	extent of the factored diagonal tile
[in]	je	column extent of the updated tile

rl_trsm_right_push_one()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverDynamic< T, Config >::rl_trsm_right_push_one ( const int n, T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT pk, const T *TDLS_RESTRICT tile, const int j0, const int ke, const int je, const int k0, T *TDLS_RESTRICT y, const int rhs_stride )

inlinestaticconstexprnoexcept

RL, column-major layout, solve_inplace: rl_trsm_right_one, then the fused U^T forward push. Akj is then the final U(k,j) tile: the solved z_k segment is pushed into the y_j segment while the tile sits in registers.

Parameters

[in]	n	system dimension
[in,out]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A
[in]	pk	physical rows of the diagonal block row
[in]	tile	factored diagonal tile
[in]	j0	first global column of the updated tile
[in]	ke	extent of the factored diagonal tile
[in]	je	column extent of the updated tile
[in]	k0	first global row/column of the step
[in,out]	y	fused right-hand side
[in]	rhs_stride	element stride of y

rl_update_row_one()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<bool fuse_rhs>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverDynamic< T, Config >::rl_update_row_one ( const int n, T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, const int *TDLS_RESTRICT pk, const T *TDLS_RESTRICT tile, const int k, const int i0, const int ke, const int ie, T *TDLS_RESTRICT y = nullptr, const int rhs_stride = 1 )

inlinestaticconstexprnoexcept

RL: TRSM down + Schur sweep of one row block below the diagonal, with the optional fused forward-substitution push.

Template Parameters

fuse_rhs

push the fused RHS y along (solve_inplace)

Parameters

[in]	n	system dimension
[in,out]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A
[in]	piv	permutation (logical -> physical row)
[in]	piv_stride	element stride of piv
[in]	pk	physical rows of the diagonal tile
[in]	tile	factored diagonal tile
[in]	k	step index (k0 = k*tile_size)
[in]	i0	first global row of the updated block row
[in]	ke	extent of the diagonal tile
[in]	ie	row extent of the updated block row
[in,out]	y	fused right-hand side (fuse_rhs only)
[in]	rhs_stride	element stride of y

solve() [1/2]

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool tdls::TiledLUppSolverDynamic< T, Config >::solve ( int n, T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride, const T *TDLS_RESTRICT b, T *TDLS_RESTRICT x, const int rhs_stride )

inlinestaticnodiscardconstexprnoexcept

Diagnostics-free solve overload: no out-of-tile out-parameter at all.

Parameters

[in]	n	system dimension (n >= 1)
[in,out]	A	on entry the matrix (pre-offset by the caller), on exit its factorization
[in]	A_stride	element stride of A
[out]	piv	permutation storage (always caller-provided)
[in]	piv_stride	element stride of piv
[in]	b	right-hand side, in original order
[out]	x	solution
[in]	rhs_stride	element stride of b and x

Returns

false on a singular matrix.

solve() [2/2]

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<bool oot_diagnostics = true>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool tdls::TiledLUppSolverDynamic< T, Config >::solve ( int n, T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride, const T *TDLS_RESTRICT b, T *TDLS_RESTRICT x, const int rhs_stride, int & oot_count )

inlinestaticnodiscardconstexprnoexcept

factorize + substitute in one call.

Template Parameters

oot_diagnostics

compile the out-of-tile counter in or out

Parameters

[in]	n	system dimension (n >= 1)
[in,out]	A	on entry the matrix (pre-offset by the caller), on exit its factorization (usable for further substitute* calls)
[in]	A_stride	element stride of A
[out]	piv	permutation storage (always caller-provided)
[in]	piv_stride	element stride of piv
[in]	b	right-hand side, in original order
[out]	x	solution
[in]	rhs_stride	element stride of b and x
[out]	oot_count	number of columns whose best in-tile pivot fell below oot_pivot_threshold

Returns

false on a singular matrix.

solve_inplace() [1/2]

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool tdls::TiledLUppSolverDynamic< T, Config >::solve_inplace ( int n, T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT y, const int rhs_stride )

inlinestaticnodiscardconstexprnoexcept

Diagnostics-free solve_inplace overload: no out-of-tile out-parameter at all.

Parameters

[in]	n	system dimension (n >= 1)
[in,out]	A	on entry the matrix (pre-offset by the caller), on exit its factorization
[in]	A_stride	element stride of A
[out]	piv	permutation storage (always caller-provided)
[in]	piv_stride	element stride of piv
[in,out]	y	right-hand side on entry, solution on exit
[in]	rhs_stride	element stride of y

Returns

false on a singular matrix (y left partially updated).

solve_inplace() [2/2]

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<bool oot_diagnostics = true>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool tdls::TiledLUppSolverDynamic< T, Config >::solve_inplace ( int n, T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT y, const int rhs_stride, int & oot_count )

inlinestaticnodiscardconstexprnoexcept

Factorization with the forward substitution folded in.

y holds the unpermuted RHS on entry and the solution on exit. Each L panel is applied to y while it sits in registers during the factorization, so the separate forward pass disappears, and all its tile reloads with it; only the backward pass remains. y follows the rows through pivoting (swap hook in factor_diag_column), which makes the result bitwise-identical to factorize + substitute_inplace under the same contraction of multiply-adds (see the file documentation). Under the column-major layout, the U panels are applied instead (U^T z = b, each diagonal tile solved as soon as it is factored), y is not swapped, and the backward pass L^T w = z is followed by x = P^T w: the same sweep kernels as substitute_inplace, and a final permutation that only moves values, hence the same bitwise identity.

Template Parameters

oot_diagnostics

compile the out-of-tile counter in or out

Parameters

[in]	n	system dimension (n >= 1)
[in,out]	A	on entry the matrix (pre-offset by the caller), on exit its factorization
[in]	A_stride	element stride of A
[out]	piv	permutation storage (always caller-provided)
[in]	piv_stride	element stride of piv
[in,out]	y	right-hand side on entry, solution on exit
[in]	rhs_stride	element stride of y
[out]	oot_count	number of columns whose best in-tile pivot fell below oot_pivot_threshold

Returns

false on a singular matrix (y left partially updated).

solve_inplace_multirhs() [1/2]

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int pass_width = 0>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool tdls::TiledLUppSolverDynamic< T, Config >::solve_inplace_multirhs ( int n, int nrhs, T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT y, const int rhs_stride, const std::size_t xcol_stride )

inlinestaticnodiscardconstexprnoexcept

Diagnostics-free solve_inplace_multirhs overload: no out-of-tile out-parameter at all.

Template Parameters

pass_width

columns solved together per substitution pass; the default 0, or any value >= nrhs, means one single pass

Parameters

[in]	n	system dimension (n >= 1)
[in]	nrhs	total number of right-hand-side columns
[in,out]	A	on entry the matrix (pre-offset by the caller), on exit its factorization
[in]	A_stride	element stride of A
[out]	piv	permutation storage (always caller-provided)
[in]	piv_stride	element stride of piv
[in,out]	y	nrhs right-hand-side columns on entry, nrhs solution columns on exit
[in]	rhs_stride	element stride of y
[in]	xcol_stride	element stride between columns of y

Returns

false on a singular matrix (y left untouched).

solve_inplace_multirhs() [2/2]

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int pass_width = 0, bool oot_diagnostics = true>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool tdls::TiledLUppSolverDynamic< T, Config >::solve_inplace_multirhs ( int n, int nrhs, T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT y, const int rhs_stride, const std::size_t xcol_stride, int & oot_count )

inlinestaticnodiscardconstexprnoexcept

factorize + substitute_inplace_multirhs in one call: y holds the nrhs unpermuted right-hand-side columns on entry and the nrhs solutions on exit.

Unlike solve_inplace, the forward pass is not folded into the factorization: a pass amortizes the forward tile loads across its columns, which removes most of what the folding saves.

Template Parameters

pass_width	columns solved together per substitution pass; the default 0, or any value >= nrhs, means one single pass
oot_diagnostics	compile the out-of-tile counter in or out

Parameters

[in]	n	system dimension (n >= 1)
[in]	nrhs	total number of right-hand-side columns
[in,out]	A	on entry the matrix (pre-offset by the caller), on exit its factorization
[in]	A_stride	element stride of A
[out]	piv	permutation storage (always caller-provided)
[in]	piv_stride	element stride of piv
[in,out]	y	nrhs right-hand-side columns on entry, nrhs solution columns on exit
[in]	rhs_stride	element stride of y
[in]	xcol_stride	element stride between columns of y
[out]	oot_count	number of columns whose best in-tile pivot fell below oot_pivot_threshold

Returns

false on a singular matrix (y left untouched).

solve_multirhs() [1/2]

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int pass_width = 0>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool tdls::TiledLUppSolverDynamic< T, Config >::solve_multirhs ( int n, int nrhs, T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride, const T *TDLS_RESTRICT b, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride )

inlinestaticnodiscardconstexprnoexcept

Diagnostics-free solve_multirhs overload: no out-of-tile out-parameter at all.

Template Parameters

pass_width

columns solved together per substitution pass; the default 0, or any value >= nrhs, means one single pass

Parameters

[in]	n	system dimension (n >= 1)
[in]	nrhs	total number of right-hand-side columns
[in,out]	A	on entry the matrix (pre-offset by the caller), on exit its factorization
[in]	A_stride	element stride of A
[out]	piv	permutation storage (always caller-provided)
[in]	piv_stride	element stride of piv
[in]	b	nrhs right-hand-side columns, in original order
[out]	x	nrhs solution columns
[in]	rhs_stride	element stride of b and x
[in]	xcol_stride	element stride between columns of b and x

Returns

false on a singular matrix.

solve_multirhs() [2/2]

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int pass_width = 0, bool oot_diagnostics = true>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool tdls::TiledLUppSolverDynamic< T, Config >::solve_multirhs ( int n, int nrhs, T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride, const T *TDLS_RESTRICT b, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride, int & oot_count )

inlinestaticnodiscardconstexprnoexcept

factorize + substitute_multirhs in one call.

Template Parameters

pass_width	columns solved together per substitution pass; the default 0, or any value >= nrhs, means one single pass
oot_diagnostics	compile the out-of-tile counter in or out

Parameters

[in]	n	system dimension (n >= 1)
[in]	nrhs	total number of right-hand-side columns
[in,out]	A	on entry the matrix (pre-offset by the caller), on exit its factorization (usable for further substitute* calls)
[in]	A_stride	element stride of A
[out]	piv	permutation storage (always caller-provided)
[in]	piv_stride	element stride of piv
[in]	b	nrhs right-hand-side columns, in original order
[out]	x	nrhs solution columns
[in]	rhs_stride	element stride of b and x
[in]	xcol_stride	element stride between columns of b and x
[out]	oot_count	number of columns whose best in-tile pivot fell below oot_pivot_threshold

Returns

false on a singular matrix.

store_tile()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverDynamic< T, Config >::store_tile ( const int n, T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT prow, const int col0, const T *TDLS_RESTRICT t, const int re, const int ce )

inlinestaticconstexprnoexcept

Store a re x ce tile through a cached physical-row segment.

Parameters

[in]	n	system dimension
[in,out]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A
[in]	prow	physical rows of the tile
[in]	col0	first global column of the tile
[in]	t	source register tile
[in]	re	tile row extent
[in]	ce	tile column extent

store_tile_piv()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverDynamic< T, Config >::store_tile_piv ( const int n, T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, const int row0, const int col0, const T *TDLS_RESTRICT t, const int re, const int ce )

inlinestaticconstexprnoexcept

Store a re x ce tile, reading the permutation inline (one read per row).

Parameters

[in]	n	system dimension
[in,out]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A
[in]	piv	permutation (logical -> physical row)
[in]	piv_stride	element stride of piv
[in]	row0	first global (logical) row of the tile
[in]	col0	first global column of the tile
[in]	t	source register tile
[in]	re	tile row extent
[in]	ce	tile column extent

substitute()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverDynamic< T, Config >::substitute ( int n, const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, const T *TDLS_RESTRICT b, T *TDLS_RESTRICT x, const int rhs_stride )

inlinestaticconstexprnoexcept

Solve x := U^-1 L^-1 P b from a prior factorize (column-major layout: x := P^T L^-T U^-T b). b and x must not alias (use substitute_inplace for the aliased single-buffer pattern).

Parameters

[in]	n	system dimension
[in]	A	factored matrix produced by factorize
[in]	A_stride	element stride of A
[in]	piv	permutation produced by factorize
[in]	piv_stride	element stride of piv
[in]	b	right-hand side, in original (unpermuted) order
[out]	x	solution
[in]	rhs_stride	element stride of b and x

substitute_canonical()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverDynamic< T, Config >::substitute_canonical ( int n, const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, const int col, T *TDLS_RESTRICT x, const int rhs_stride )

inlinestaticconstexprnoexcept

Solve with b = e_col generated on the fly: the consistent-tangent-operator path.

Thin alias of the canonical pass engine below with nrhs = 1, which collapses to exactly the single-column code, so there is no separate implementation to maintain.

Parameters

[in]	n	system dimension
[in]	A	factored matrix produced by factorize
[in]	A_stride	element stride of A
[in]	piv	permutation produced by factorize
[in]	piv_stride	element stride of piv
[in]	col	index of the canonical column e_col
[out]	x	solution
[in]	rhs_stride	element stride of x

substitute_canonical_multirhs()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int pass_width = 0>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverDynamic< T, Config >::substitute_canonical_multirhs ( int n, int nrhs, const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, const int col0, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride )

inlinestaticconstexprnoexcept

nrhs canonical columns e_col0 .. e_{col0+nrhs-1} solved from a prior factorize: the columns of the inverse that a consistent tangent operator needs.

Column w lives at x + w*xcol_stride. The columns are solved in passes of pass_width columns; within a pass, every L/U tile is loaded once for all the columns of the pass. Per-column arithmetic is identical whatever the cutting, and identical to nrhs separate substitute_canonical calls: results match them bitwise under the same contraction of multiply-adds (see the file documentation; nrhs = 1 collapses to substitute_canonical exactly).

Template Parameters

pass_width

columns solved together per pass; the default 0, or any value >= nrhs, means one single pass of nrhs columns

Parameters

[in]	n	system dimension
[in]	nrhs	number of consecutive canonical columns
[in]	A	factored matrix produced by factorize
[in]	A_stride	element stride of A
[in]	piv	permutation produced by factorize
[in]	piv_stride	element stride of piv
[in]	col0	index of the first canonical column
[out]	x	nrhs solution columns
[in]	rhs_stride	element stride of x
[in]	xcol_stride	element stride between columns of x

substitute_canonical_multirhs_pass()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverDynamic< T, Config >::substitute_canonical_multirhs_pass ( const int n, const int nrhs, const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, const int col0, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride )

inlinestaticconstexprnoexcept

One pass of the canonical multi right-hand-side substitution: nrhs canonical columns e_col0 .. e_{col0+nrhs-1} generated in permuted order into x, then one triangular sweep for all nrhs columns together. Under the column-major layout the columns are generated unpermuted, swept through U^T then L^T, then permuted (x = P^T w). Internal engine of substitute_canonical_multirhs.

Parameters

[in]	n	system dimension
[in]	nrhs	number of columns of this pass
[in]	A	factored matrix produced by factorize
[in]	A_stride	element stride of A
[in]	piv	permutation produced by factorize
[in]	piv_stride	element stride of piv
[in]	col0	index of the first canonical column
[out]	x	nrhs solution columns
[in]	rhs_stride	element stride of x
[in]	xcol_stride	element stride between columns of x

substitute_inplace()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverDynamic< T, Config >::substitute_inplace ( int n, const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride )

inlinestaticconstexprnoexcept

Solve in place: x already holds the unpermuted RHS on entry and the solution on exit.

The permutation is applied in place by cycle decomposition (see permute_inplace). Under the column-major layout, the sweeps run first on the unpermuted x and the inverse rotation (x = P^T w) comes last (see unpermute_inplace).

Parameters

[in]	n	system dimension
[in]	A	factored matrix produced by factorize
[in]	A_stride	element stride of A
[in]	piv	permutation produced by factorize
[in]	piv_stride	element stride of piv
[in,out]	x	right-hand side on entry, solution on exit
[in]	rhs_stride	element stride of x

substitute_inplace_multirhs()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int pass_width = 0>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverDynamic< T, Config >::substitute_inplace_multirhs ( int n, int nrhs, const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride )

inlinestaticconstexprnoexcept

nrhs columns solved in place: x holds the nrhs unpermuted right-hand-side columns on entry and the nrhs solutions on exit.

The permutation is applied to each column by the cycle decomposition of permute_inplace (under the column-major layout, after the sweeps, by that of unpermute_inplace). The columns are solved in passes of pass_width columns; within a pass, every L/U tile is loaded once for all the columns of the pass. Per-column values are identical whatever the cutting, and match nrhs separate substitute_inplace calls bitwise under the same contraction of multiply-adds (see the file documentation; nrhs = 1 collapses to substitute_inplace exactly).

Template Parameters

pass_width

columns solved together per pass; the default 0, or any value >= nrhs, means one single pass of nrhs columns

Parameters

[in]	n	system dimension
[in]	nrhs	total number of right-hand-side columns
[in]	A	factored matrix produced by factorize
[in]	A_stride	element stride of A
[in]	piv	permutation produced by factorize
[in]	piv_stride	element stride of piv
[in,out]	x	nrhs right-hand-side columns on entry, nrhs solution columns on exit
[in]	rhs_stride	element stride of x
[in]	xcol_stride	element stride between columns of x

substitute_inplace_multirhs_pass()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverDynamic< T, Config >::substitute_inplace_multirhs_pass ( const int n, const int nrhs, const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride )

inlinestaticconstexprnoexcept

One pass of the in-place multi right-hand-side substitution: nrhs columns of x permuted by the cycle decomposition of permute_inplace, then one triangular sweep for all nrhs columns together. Under the column-major layout the sweeps come first, then the cycle decomposition of unpermute_inplace applies x = P^T w. Internal engine of substitute_inplace_multirhs.

Parameters

[in]	n	system dimension
[in]	nrhs	number of columns of this pass
[in]	A	factored matrix produced by factorize
[in]	A_stride	element stride of A
[in]	piv	permutation produced by factorize
[in]	piv_stride	element stride of piv
[in,out]	x	nrhs right-hand-side columns on entry, nrhs solution columns on exit
[in]	rhs_stride	element stride of x
[in]	xcol_stride	element stride between columns of x

substitute_multirhs()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int pass_width = 0>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverDynamic< T, Config >::substitute_multirhs ( int n, int nrhs, const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, const T *TDLS_RESTRICT b, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride )

inlinestaticconstexprnoexcept

Solve nrhs right-hand-side columns from a prior factorize: X := U^-1 L^-1 P B (column-major layout: X := P^T L^-T U^-T B), column by column. b and x must not alias (use substitute_inplace_multirhs for the aliased single-buffer pattern).

Column w lives at b/x + w*xcol_stride; b and x share both strides, as they share rhs_stride in substitute. The columns are solved in passes of pass_width columns; within a pass, every L/U tile is loaded once for all the columns of the pass. Per-column arithmetic is identical whatever the cutting, and identical to nrhs separate substitute calls: results match them bitwise under the same contraction of multiply-adds (see the file documentation; nrhs = 1 collapses to substitute exactly).

Template Parameters

pass_width

columns solved together per pass; the default 0, or any value >= nrhs, means one single pass of nrhs columns

Parameters

[in]	n	system dimension
[in]	nrhs	total number of right-hand-side columns
[in]	A	factored matrix produced by factorize
[in]	A_stride	element stride of A
[in]	piv	permutation produced by factorize
[in]	piv_stride	element stride of piv
[in]	b	nrhs right-hand-side columns, in original (unpermuted) order
[out]	x	nrhs solution columns
[in]	rhs_stride	element stride of b and x
[in]	xcol_stride	element stride between columns of b and x

substitute_multirhs_pass()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverDynamic< T, Config >::substitute_multirhs_pass ( const int n, const int nrhs, const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, const T *TDLS_RESTRICT b, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride )

inlinestaticconstexprnoexcept

One pass of the multi right-hand-side substitution: nrhs columns of b gathered in permuted order into x, then one triangular sweep for all nrhs columns together. Under the column-major layout the columns are copied unpermuted, swept through U^T then L^T, then permuted (x = P^T w). Internal engine of substitute_multirhs.

Parameters

[in]	n	system dimension
[in]	nrhs	number of columns of this pass
[in]	A	factored matrix produced by factorize
[in]	A_stride	element stride of A
[in]	piv	permutation produced by factorize
[in]	piv_stride	element stride of piv
[in]	b	nrhs right-hand-side columns, in original (unpermuted) order
[out]	x	nrhs solution columns
[in]	rhs_stride	element stride of b and x
[in]	xcol_stride	element stride between columns of b and x

tile_size_at()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr int tdls::TiledLUppSolverDynamic< T, Config >::tile_size_at ( const int t0, const int n )

inlinestaticconstexprnoexcept

Extent of the tile starting at row/column t0: tile_size, or less for the last tile.

Parameters

[in]	t0	first global row/column of the tile
[in]	n	system dimension

Returns

min(tile_size, n - t0)

transposed_bwd_only()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverDynamic< T, Config >::transposed_bwd_only ( const int n, const int nrhs, const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride )

inlinestaticconstexprnoexcept

Transposed backward pass alone, used by transposed_fwd_bwd and by solve_inplace (whose forward pass happens inside the factorization).

Parameters

[in]	n	system dimension
[in]	nrhs	number of columns processed together
[in]	A	factored matrix (caller-pre-offset)
[in]	A_stride	element stride of A
[in]	piv	permutation from factorize
[in]	piv_stride	element stride of piv
[in,out]	x	solution column(s)
[in]	rhs_stride	element stride of x
[in]	xcol_stride	element stride between columns of x

transposed_bwd_step()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverDynamic< T, Config >::transposed_bwd_step ( const int n, const int nrhs, const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride, const int k )

inlinestaticconstexprnoexcept

Transposed backward step: pull the trailing contributions with L^T, then L^T-solve the diagonal tile's segment.

Parameters

[in]	n	system dimension
[in]	nrhs	number of columns processed together
[in]	A	factored matrix (caller-pre-offset)
[in]	A_stride	element stride of A
[in]	piv	permutation from factorize
[in]	piv_stride	element stride of piv
[in,out]	x	solution column(s)
[in]	rhs_stride	element stride of x
[in]	xcol_stride	element stride between columns of x
[in]	k	step index (k0 = k*tile_size)

transposed_fwd_bwd()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverDynamic< T, Config >::transposed_fwd_bwd ( const int n, const int nrhs, const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride )

inlinestaticconstexprnoexcept

Transposed triangular sweeps on unpermuted column(s) x: U^T z = x, then L^T w = z, w left in x (x = P^T w is the caller's last step).

Parameters

[in]	n	system dimension
[in]	nrhs	number of columns processed together
[in]	A	factored matrix (caller-pre-offset)
[in]	A_stride	element stride of A
[in]	piv	permutation from factorize
[in]	piv_stride	element stride of piv
[in,out]	x	right-hand side column(s) on entry, w on exit
[in]	rhs_stride	element stride of x
[in]	xcol_stride	element stride between columns of x

transposed_fwd_step()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverDynamic< T, Config >::transposed_fwd_step ( const int n, const int nrhs, const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride, const int k )

inlinestaticconstexprnoexcept

Transposed forward step: U^T-solve the diagonal tile's segment, then push it into the following segments.

Parameters

[in]	n	system dimension
[in]	nrhs	number of columns processed together
[in]	A	factored matrix (caller-pre-offset)
[in]	A_stride	element stride of A
[in]	piv	permutation from factorize
[in]	piv_stride	element stride of piv
[in,out]	x	right-hand side column(s)
[in]	rhs_stride	element stride of x
[in]	xcol_stride	element stride between columns of x
[in]	k	step index (k0 = k*tile_size)

transposed_pull_one()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverDynamic< T, Config >::transposed_pull_one ( const T *TDLS_RESTRICT l, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride, const int nrhs, const int k0, const int m0, const int ke, const int me )

inlinestaticconstexprnoexcept

Backward pull with L^T: x_k segment -= L(m,k)^T * x_m.

Parameters

[in]	l	L(m,k) tile (me rows, ke columns)
[in,out]	x	solution column(s)
[in]	rhs_stride	element stride of x
[in]	xcol_stride	element stride between columns of x
[in]	nrhs	number of columns processed together
[in]	k0	first global index of the updated segment
[in]	m0	first global index of the trailing segment
[in]	ke	extent of the updated segment's tile
[in]	me	extent of the trailing segment's tile

transposed_push_one()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverDynamic< T, Config >::transposed_push_one ( const T *TDLS_RESTRICT u, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride, const int nrhs, const int k0, const int m0, const int ke, const int me )

inlinestaticconstexprnoexcept

Forward push with U^T: x_m segment -= U(k,m)^T * x_k.

Parameters

[in]	u	U(k,m) tile (ke rows, me columns)
[in,out]	x	right-hand side column(s)
[in]	rhs_stride	element stride of x
[in]	xcol_stride	element stride between columns of x
[in]	nrhs	number of columns processed together
[in]	k0	first global index of the solved segment
[in]	m0	first global index of the target segment
[in]	ke	extent of the solved segment's tile
[in]	me	extent of the target segment's tile

transposed_solve_tile_lower_unit()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverDynamic< T, Config >::transposed_solve_tile_lower_unit ( const T *TDLS_RESTRICT l, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride, const int nrhs, const int k0, const int ke )

inlinestaticconstexprnoexcept

In-tile backward solve with L^T (unit diagonal).

Parameters

[in]	l	strictly lower part of the diagonal tile
[in,out]	x	solution column(s)
[in]	rhs_stride	element stride of x
[in]	xcol_stride	element stride between columns of x
[in]	nrhs	number of columns processed together
[in]	k0	first global index of the segment
[in]	ke	extent of the diagonal tile

transposed_solve_tile_upper()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverDynamic< T, Config >::transposed_solve_tile_upper ( const T *TDLS_RESTRICT t, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride, const int nrhs, const int k0, const int ke )

inlinestaticconstexprnoexcept

In-tile forward solve with U^T (the diagonal of the tile holds the pivot reciprocals).

Parameters

[in]	t	factored diagonal tile
[in,out]	x	right-hand side column(s)
[in]	rhs_stride	element stride of x
[in]	xcol_stride	element stride between columns of x
[in]	nrhs	number of columns processed together
[in]	k0	first global index of the segment
[in]	ke	extent of the diagonal tile

unpermute_inplace()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverDynamic< T, Config >::unpermute_inplace ( const int n, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride )

inlinestaticconstexprnoexcept

x := P^T x in place for one strided column (x[piv[i]] receives x[i]): the inverse rotation of permute_inplace, each cycle carried once from its smallest index, a 64-bit bitmask up to n = 64, a cycle-leader scan beyond. Values are moved, never computed.

Parameters

[in]	n	system dimension
[in]	piv	permutation from factorize
[in]	piv_stride	element stride of piv
[in,out]	x	w on entry, the solution on exit
[in]	rhs_stride	element stride of x

Member Data Documentation

schedule

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

Schedule tdls::TiledLUppSolverDynamic< T, Config >::schedule

staticconstexpr

Initial value:

=
        Config.schedule

elimination schedule (RightLooking or LeftLooking)

---

The documentation for this struct was generated from the following file:

• include/tdls/solvers/tiled_lupp/solver_dynamic.hpp