tdls::TiledLUppSolverStatic< T, N, Config > Struct Template Reference

Tiled dense LU factorization with logical partial pivoting and out-of-tile pivot recovery, solving one NxN system per call. More...

#include <solver_static.hpp>

Public Types
using	Operations = TiledLUppTileOperations<T, tile_size, Config.unroll_loops>
	Tile micro-kernels instantiated for this configuration.

Static Public Member Functions
template<int row_extent, int col_extent, bool internal_matrix>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	load_tile (const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT prow, const int col0, T *TDLS_RESTRICT t) noexcept
	Load an RxC tile through a cached physical-row segment.
template<int row_extent, int col_extent, bool internal_matrix>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	store_tile (T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT prow, const int col0, const T *TDLS_RESTRICT t) noexcept
	Store an RxC tile through a cached physical-row segment.
template<int row_extent, int col_extent, bool internal_piv, bool internal_matrix>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	load_tile_piv (const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, const int row0, const int col0, T *TDLS_RESTRICT t) noexcept
	Load an RxC tile, reading the permutation inline (one read per row).
template<int row_extent, int col_extent, bool internal_piv, bool internal_matrix>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	store_tile_piv (T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, const int row0, const int col0, const T *TDLS_RESTRICT t) noexcept
	Store an RxC tile, reading the permutation inline (one read per row).
template<int row_extent, bool internal_piv, bool internal_matrix>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	load_tile_piv_lower (const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, const int row0, const int col0, T *TDLS_RESTRICT t) noexcept
	Triangular variant of the diagonal-tile load for the forward substitution (the backward one under the column-major layout): only the strict lower triangle of L\U is read.
template<int row_extent, bool internal_piv, bool internal_matrix>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	load_tile_piv_upper (const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, const int row0, const int col0, T *TDLS_RESTRICT t) noexcept
	Triangular variant of the diagonal-tile load for the backward substitution (the forward one under the column-major layout): only the upper triangle including the diagonal is read.
template<int k_extent, bool internal_piv, bool internal_matrix, bool oot_diagnostics, bool fuse_rhs, bool internal_rhs>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool	factor_diag_column (T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride, const int k0, T *TDLS_RESTRICT tile, int &oot_count, const int c, T *TDLS_RESTRICT y, const int rhs_stride) noexcept
	One column step of the diagonal-tile factorization: pivot search (in-tile, then out-of-tile recovery), permutation update, row swap or cross-tile pull, column elimination.
template<int k_extent, bool internal_piv, bool internal_matrix, bool oot_diagnostics, bool fuse_rhs = false, bool internal_rhs = true>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool	factor_diag_tile (T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride, const int k0, T *TDLS_RESTRICT tile, int &oot_count, T *TDLS_RESTRICT y=nullptr, const int rhs_stride=1) noexcept
	Factor the KExKE diagonal tile in registers, with out-of-tile pivot recovery (drives the per-column loop of factor_diag_column).
template<int k_extent, int j_extent, bool internal_matrix>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	rl_trsm_right_one (T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT pk, const T *TDLS_RESTRICT tile, const int j0) noexcept
	RL: one row-panel tile update, Akj := L^-1 Akj.
template<int k_extent, int j_extent, bool internal_matrix, bool internal_rhs>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	rl_trsm_right_push_one (T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT pk, const T *TDLS_RESTRICT tile, const int j0, const int k0, T *TDLS_RESTRICT y, const int rhs_stride) noexcept
	RL, column-major layout, solve_inplace: rl_trsm_right_one, then the fused U^T forward push. Akj is then the final U(k,j) tile: the solved z_k segment is pushed into the y_j segment while the tile sits in registers.
template<int k_extent, int i_extent, int j_extent, bool internal_matrix>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	rl_schur_one (T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT pk, const int *TDLS_RESTRICT pi, const T *TDLS_RESTRICT Aik, const int j0) noexcept
	RL: one Schur-complement tile update, Aij -= Aik * Akj, streaming the factored Akj row by row from remote memory (it is not worth a third register tile).
template<int k_extent, int i_extent, bool internal_piv, bool internal_matrix, bool fuse_rhs = false, bool internal_rhs = true>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	rl_update_row_one (T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, const int *TDLS_RESTRICT pk, const T *TDLS_RESTRICT tile, const int k, const int i0, T *TDLS_RESTRICT y=nullptr, const int rhs_stride=1) noexcept
	RL: TRSM down + Schur sweep of one row block below the diagonal, with the optional fused forward-substitution push.
template<int k_extent, bool internal_piv, bool internal_matrix, bool oot_diagnostics, bool fuse_rhs = false, bool internal_rhs = true>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool	rl_step (T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride, const int k, int &oot_count, T *TDLS_RESTRICT y=nullptr, const int rhs_stride=1) noexcept
	RL: one full factorization step (diagonal tile + trailing updates).
template<int row_extent, int col_extent, bool internal_piv, bool internal_matrix>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	ll_correct_tile (const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, const int row0, const int col0, const int k0, T *TDLS_RESTRICT t) noexcept
	LL: t (RExCE, rows row0.., cols col0..) -= sum over prior tiles bj < k0/tile_size of L(row0.., bj) * U(bj, col0..). All prior tiles are full.
template<int k_extent, int i_extent, bool internal_piv, bool internal_matrix, bool fuse_rhs = false, bool internal_rhs = true>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	ll_update_below_one (T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, const T *TDLS_RESTRICT tile, const int k0, const int i0, T *TDLS_RESTRICT y=nullptr, const int rhs_stride=1) noexcept
	LL: correct + TRSM one L-panel tile below the diagonal, with the optional fused forward-substitution push.
template<int k_extent, int j_extent, bool internal_piv, bool internal_matrix>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	ll_update_right_one (T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, const T *TDLS_RESTRICT tile, const int k0, const int j0) noexcept
	LL: correct + TRSM one U-panel tile right of the diagonal.
template<int k_extent, int j_extent, bool internal_piv, bool internal_matrix, bool internal_rhs>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	ll_update_right_push_one (T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, const T *TDLS_RESTRICT tile, const int k0, const int j0, T *TDLS_RESTRICT y, const int rhs_stride) noexcept
	LL, column-major layout, solve_inplace: ll_update_right_one, then the fused U^T forward push. B is then the final U(k,j) tile: the solved z_k segment is pushed into the y_j segment while the tile sits in registers.
template<int k_extent, bool internal_piv, bool internal_matrix, bool oot_diagnostics, bool fuse_rhs = false, bool internal_rhs = true>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool	ll_step (T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride, const int k, int &oot_count, T *TDLS_RESTRICT y=nullptr, const int rhs_stride=1) noexcept
	LL: one full factorization step (correct + factor the diagonal tile, then its L and U panels).
template<int w, bool internal_rhs>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	physical_interchange_rhs (T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride, const int j, const int r) noexcept
	Interchanges rows j and r > j of the w columns of x (physical scheme): a test of every unrolled row index against r.
template<int n>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	physical_interchange_local (T *TDLS_RESTRICT X, const int j, const int r, const int cs, const int rs) noexcept
	Interchanges rows j and r > j of the n columns of X, entry (i, c) at i * rs + c * cs, X a caller-local array (physical scheme): a test of every unrolled row index against r. Written with the strides as arguments, as the former register path of ArrowheadLUpp wrote it: through the addressing macros, nvcc dispatches the interchanges of several passes through more jump tables, 4 % slower on average on an RTX 5080.
template<int w, bool internal_rhs, bool internal_piv, bool reverse>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	physical_permute (const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride) noexcept
	Applies the interchanges to w columns of x (physical scheme): in their order (P x), or in reverse order (P^T x).
template<bool internal_piv, bool internal_matrix, bool check>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool	physical_step (T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride, const int j, const T u, bool &singular, int &weak) noexcept
	Step j of the physical factorization: the pivot of column j by the rule of a single tile (threshold pivoting, the singularity floor), the interchange of the whole rows, the elimination below the pivot. Branch-free over the steps: a column without a candidate above singular_floor sets singular and the elimination goes on with a zero reciprocal, so that the unrolled steps run as straight-line code.
template<bool internal_piv, bool internal_matrix>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool	physical_factorize (T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride, int &weak) noexcept
	Physical factorization: P S = L U in place, L below the diagonal and U above it in the pivoted row order, 1 / pivot on the diagonal; piv[j] the row exchanged with row j at step j.
template<bool internal_piv, bool internal_matrix>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr int	physical_factorize_checked (T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride, const T u) noexcept
	Physical factorization with its multipliers bounded, the check fused into the elimination (the condensation of ArrowheadLUpp under RowInterchange::Physical): as physical_factorize, and every multiplier of L checked against 1 / u.
template<int w, bool internal_rhs, bool internal_piv, bool internal_matrix>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	physical_forward (const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride) noexcept
	First stage of the physical substitution of w columns: x := L^-1 P x under the row-major layout (the interchanges in their order, then the columns of L), x := U^-T x under the column-major one.
template<int w, bool internal_rhs, bool internal_piv, bool internal_matrix>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	physical_backward (const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride) noexcept
	Second stage of the physical substitution of w columns: x := U^-1 x under the row-major layout, x := P^T L^-T x under the column-major one (the rows of L^T, then the interchanges in reverse order).
template<int nrhs, int pass_width, int stages, bool internal_rhs, bool internal_piv, bool internal_matrix>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	physical_multirhs (const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride) noexcept
	Physical substitution of nrhs columns in passes of pass_width columns: the first stage, the second, or both.
template<bool internal_piv, bool internal_matrix, bool oot_diagnostics = true, bool fuse_rhs = false, bool internal_rhs = true, int A_static_stride = dynamic_stride, int piv_static_stride = dynamic_stride, int rhs_static_stride = dynamic_stride>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool	factorize (T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride, int &oot_count, T *TDLS_RESTRICT y=nullptr, const int rhs_stride=1) noexcept
	Factor A := P*L*U in place.
template<bool internal_piv, bool internal_matrix, int A_static_stride = dynamic_stride, int piv_static_stride = dynamic_stride>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool	factorize (T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride) noexcept
	Diagnostics-free factorize overload: no out-of-tile out-parameter at all.
template<int k_extent, int m_extent, int pass_width, bool internal_rhs, bool internal_piv, bool internal_matrix>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	fwd_push_one (const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride, const int k0, const int m0) noexcept
	Forward push: subtract L(m,k) * x_k from the x_m segment, for pass_width columns at once.
template<int k_extent, int pass_width, bool internal_rhs, bool internal_piv, bool internal_matrix>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	fwd_step (const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride, const int k) noexcept
	Forward step: unit-lower solve the diagonal tile's segment, then push it into the tiles below.
template<int k_extent, int m_extent, int pass_width, bool internal_rhs, bool internal_piv, bool internal_matrix>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	bwd_pull_one (const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride, const int k0, const int m0) noexcept
	Backward pull: subtract U(k,m) * x_m from the x_k segment, for pass_width columns at once.
template<int k_extent, int pass_width, bool internal_rhs, bool internal_piv, bool internal_matrix>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	bwd_step (const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride, const int k) noexcept
	Backward step: pull the trailing contributions, then upper-solve the diagonal tile's segment.
template<int pass_width, bool internal_rhs, bool internal_piv, bool internal_matrix>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	bwd_only (const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride) noexcept
	Backward pass alone, used by fwd_bwd and by solve_inplace (whose forward pass happens inside the factorization).
template<int pass_width, bool internal_rhs, bool internal_piv, bool internal_matrix>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	fwd_bwd (const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride) noexcept
	Triangular solves on already-permuted column(s) x. pass_width columns are processed per tile visit, so every L/U tile is loaded once for the whole block instead of once per column (pass_width = 1 is the single-RHS case).
template<int k_extent, int pass_width, bool internal_rhs>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	transposed_solve_tile_upper (const T *TDLS_RESTRICT t, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride, const int k0) noexcept
	In-tile forward solve with U^T: x_k segment := U(k,k)^-T x_k (the diagonal of the tile holds the pivot reciprocals).
template<int k_extent, int m_extent, int pass_width, bool internal_rhs>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	transposed_push_one (const T *TDLS_RESTRICT u, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride, const int k0, const int m0) noexcept
	Forward push with U^T: x_m segment -= U(k,m)^T * x_k.
template<int k_extent, int m_extent, int pass_width, bool internal_rhs>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	transposed_pull_one (const T *TDLS_RESTRICT l, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride, const int k0, const int m0) noexcept
	Backward pull with L^T: x_k segment -= L(m,k)^T * x_m.
template<int k_extent, int pass_width, bool internal_rhs>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	transposed_solve_tile_lower_unit (const T *TDLS_RESTRICT l, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride, const int k0) noexcept
	In-tile backward solve with L^T (unit diagonal: no division and no multiplication by the diagonal).
template<int k_extent, int pass_width, bool internal_rhs, bool internal_piv, bool internal_matrix>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	transposed_fwd_step (const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride, const int k) noexcept
	Transposed forward step: U^T-solve the diagonal tile's segment, then push it into the following segments.
template<int k_extent, int pass_width, bool internal_rhs, bool internal_piv, bool internal_matrix>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	transposed_bwd_step (const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride, const int k) noexcept
	Transposed backward step: pull the trailing contributions with L^T, then L^T-solve the diagonal tile's segment.
template<int pass_width, bool internal_rhs, bool internal_piv, bool internal_matrix>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	transposed_bwd_only (const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride) noexcept
	Transposed backward pass alone, used by transposed_fwd_bwd and by solve_inplace (whose forward pass happens inside the factorization).
template<int pass_width, bool internal_rhs, bool internal_piv, bool internal_matrix>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	transposed_fwd_bwd (const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride) noexcept
	Transposed triangular sweeps on unpermuted column(s) x: U^T z = x, then L^T w = z, w left in x (x = P^T w is the caller's last step).
template<int pass_width, bool internal_rhs, bool internal_piv>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	unpermute_cycles (const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride) noexcept
	x := P^T x in place (x[piv[i]] receives x[i]), the last step of the transposed solve, by cycle decomposition: the inverse of the rotation of substitute_inplace, each cycle carried once from its smallest index. Up to N = 64 a bitmask tracks the visited rows; beyond, a cycle-leader scan. Values are moved, never computed.
template<int pass_width, bool internal_piv>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	unpermute_predicated (const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x) noexcept
	x := P^T x for an internal x, predicated over the unrolled index range: x must never be dynamically indexed or it is demoted to local memory. Written as the gather of substitute, a scalar selected over the sources (x[j] receives w[i] where piv[i] == j), which compiles to selects; a predicated store into x would compile to branches, divergent across the threads of a warp.
template<int pass_width, bool internal_rhs, bool internal_piv>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	unpermute (const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride) noexcept
	x := P^T x, predicated for an internal x, by cycles for a remote one.
template<bool internal_rhs, bool internal_piv, bool internal_matrix, int A_static_stride = dynamic_stride, int piv_static_stride = dynamic_stride, int rhs_static_stride = dynamic_stride>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	substitute (const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, const T *TDLS_RESTRICT b, T *TDLS_RESTRICT x, const int rhs_stride) noexcept
	Solve x := U^-1 L^-1 P b from a prior factorize (column-major layout: x := P^T L^-T U^-T b). b and x must not alias (use substitute_inplace for the aliased single-buffer pattern).
template<bool internal_rhs, bool internal_piv, bool internal_matrix, int A_static_stride = dynamic_stride, int piv_static_stride = dynamic_stride, int rhs_static_stride = dynamic_stride>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	substitute_canonical (const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, const int col, T *TDLS_RESTRICT x, const int rhs_stride) noexcept
	Solve with b = e_col generated on the fly: the consistent-tangent-operator path.
template<int pass_width, bool internal_rhs, bool internal_piv, bool internal_matrix>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	substitute_canonical_multirhs_pass (const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, const int col0, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride) noexcept
	One pass of the canonical multi right-hand-side substitution: pass_width canonical columns e_col0 .. e_{col0+pass_width-1} generated in permuted order into x, then one triangular sweep for all pass_width columns together. Under the column-major layout the columns are generated unpermuted, swept through U^T then L^T, then permuted (x = P^T w). Internal engine of substitute_canonical_multirhs.
template<int nrhs, bool internal_rhs, bool internal_piv, bool internal_matrix, int pass_width = 0, int A_static_stride = dynamic_stride, int piv_static_stride = dynamic_stride, int rhs_static_stride = dynamic_stride>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	substitute_canonical_multirhs (const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, const int col0, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride) noexcept
	nrhs canonical columns e_col0 .. e_{col0+nrhs-1} solved from a prior factorize: the columns of the inverse that a consistent tangent operator needs.
template<int pass_width, bool internal_rhs, bool internal_piv, bool internal_matrix>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	substitute_multirhs_pass (const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, const T *TDLS_RESTRICT b, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride) noexcept
	One pass of the multi right-hand-side substitution: pass_width columns of b gathered in permuted order into x, then one triangular sweep for all pass_width columns together. Under the column-major layout the columns are copied unpermuted, swept through U^T then L^T, then permuted (x = P^T w). Internal engine of substitute_multirhs.
template<int nrhs, bool internal_rhs, bool internal_piv, bool internal_matrix, int pass_width = 0, int A_static_stride = dynamic_stride, int piv_static_stride = dynamic_stride, int rhs_static_stride = dynamic_stride>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	substitute_multirhs (const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, const T *TDLS_RESTRICT b, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride) noexcept
	Solve nrhs right-hand-side columns from a prior factorize: X := U^-1 L^-1 P B (column-major layout: X := P^T L^-T U^-T B), column by column. b and x must not alias (use substitute_inplace_multirhs for the aliased single-buffer pattern).
template<bool internal_rhs, bool internal_piv, bool internal_matrix, int A_static_stride = dynamic_stride, int piv_static_stride = dynamic_stride, int rhs_static_stride = dynamic_stride>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	substitute_inplace (const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride) noexcept
	Solve in place: x already holds the unpermuted RHS on entry and the solution on exit.
template<int pass_width, bool internal_rhs, bool internal_piv>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_ALL constexpr void	permute_cycles (const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride) noexcept
	x := P x in place for pass_width columns (row i receives row piv[i]), by the cycle decomposition of substitute_inplace, each move carrying the pass_width column entries of its row: the permutation of substitute_inplace_multirhs_pass and forward_inplace_multirhs, the inverse of unpermute_cycles. Forced inline on CPU too (TDLS_FORCE_INLINE_ALL): an out-of-line call costs 20 to 27 % under gcc -O2, and inlined it leaves the code of substitute_inplace_multirhs as it was when these lines sat in it.
template<int pass_width, bool internal_rhs, bool internal_piv, bool internal_matrix>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	substitute_inplace_multirhs_pass (const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride) noexcept
	One pass of the in-place multi right-hand-side substitution: pass_width columns of x permuted by the cycle decomposition of substitute_inplace (each move carrying the pass_width column entries of its row), then one triangular sweep for all pass_width columns together. Under the column-major layout the sweeps come first, then the cycle decomposition applies x = P^T w. Internal engine of substitute_inplace_multirhs.
template<int pass_width, bool internal_rhs, bool internal_piv, bool internal_matrix>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	forward_inplace_multirhs_pass (const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride) noexcept
	Forward stage of one pass of substitute_inplace_multirhs: the permutation and the sweep with L under the row-major layout (x := L^-1 P x), the sweep with U^T under the column-major one (x := U^-T x). Internal engine of forward_inplace_multirhs.
template<int pass_width, bool internal_rhs, bool internal_piv, bool internal_matrix>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	backward_inplace_multirhs_pass (const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride) noexcept
	Backward stage of one pass of substitute_inplace_multirhs: the sweep with U under the row-major layout (x := U^-1 x), the sweep with L^T then the inverse permutation under the column-major one (x := P^T L^-T x). Internal engine of backward_inplace_multirhs.
template<int nrhs, bool internal_rhs, bool internal_piv, bool internal_matrix, int pass_width = 0, int A_static_stride = dynamic_stride, int piv_static_stride = dynamic_stride, int rhs_static_stride = dynamic_stride>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	substitute_inplace_multirhs (const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride) noexcept
	nrhs columns solved in place: x holds the nrhs unpermuted right-hand-side columns on entry and the nrhs solutions on exit.
template<int nrhs, bool internal_rhs, bool internal_piv, bool internal_matrix, int pass_width = 0, int A_static_stride = dynamic_stride, int piv_static_stride = dynamic_stride, int rhs_static_stride = dynamic_stride>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	forward_inplace_multirhs (const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride) noexcept
	First of the two stages of substitute_inplace_multirhs, in passes of pass_width columns: x := L^-1 P x under the row-major layout, x := U^-T x under the column-major one. The caller may read or update the intermediate columns, then backward_inplace_multirhs completes the solve; the two stages in a row match substitute_inplace_multirhs bitwise.
template<int nrhs, bool internal_rhs, bool internal_piv, bool internal_matrix, int pass_width = 0, int A_static_stride = dynamic_stride, int piv_static_stride = dynamic_stride, int rhs_static_stride = dynamic_stride>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	backward_inplace_multirhs (const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride) noexcept
	Second of the two stages of substitute_inplace_multirhs, in passes of pass_width columns: x := U^-1 x under the row-major layout, x := P^T L^-T x under the column-major one.
template<bool internal_rhs, bool internal_piv, bool internal_matrix, bool oot_diagnostics = true, int A_static_stride = dynamic_stride, int piv_static_stride = dynamic_stride, int rhs_static_stride = dynamic_stride>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool	solve (T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride, const T *TDLS_RESTRICT b, T *TDLS_RESTRICT x, const int rhs_stride, int &oot_count) noexcept
	factorize + substitute in one call.
template<bool internal_rhs, bool internal_piv, bool internal_matrix, int A_static_stride = dynamic_stride, int piv_static_stride = dynamic_stride, int rhs_static_stride = dynamic_stride>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool	solve (T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride, const T *TDLS_RESTRICT b, T *TDLS_RESTRICT x, const int rhs_stride) noexcept
	Diagnostics-free solve overload: no out-of-tile out-parameter at all.
template<int nrhs, bool internal_rhs, bool internal_piv, bool internal_matrix, int pass_width = 0, bool oot_diagnostics = true, int A_static_stride = dynamic_stride, int piv_static_stride = dynamic_stride, int rhs_static_stride = dynamic_stride>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool	solve_multirhs (T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride, const T *TDLS_RESTRICT b, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride, int &oot_count) noexcept
	factorize + substitute_multirhs in one call.
template<int nrhs, bool internal_rhs, bool internal_piv, bool internal_matrix, int pass_width = 0, int A_static_stride = dynamic_stride, int piv_static_stride = dynamic_stride, int rhs_static_stride = dynamic_stride>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool	solve_multirhs (T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride, const T *TDLS_RESTRICT b, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride) noexcept
	Diagnostics-free solve_multirhs overload: no out-of-tile out-parameter at all.
template<bool internal_rhs, bool internal_piv, bool internal_matrix, bool oot_diagnostics = true, int A_static_stride = dynamic_stride, int piv_static_stride = dynamic_stride, int rhs_static_stride = dynamic_stride>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool	solve_inplace (T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT y, const int rhs_stride, int &oot_count) noexcept
	Factorization with the forward substitution folded in.
template<bool internal_rhs, bool internal_piv, bool internal_matrix, int A_static_stride = dynamic_stride, int piv_static_stride = dynamic_stride, int rhs_static_stride = dynamic_stride>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool	solve_inplace (T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT y, const int rhs_stride) noexcept
	Diagnostics-free solve_inplace overload: no out-of-tile out-parameter at all.
template<int nrhs, bool internal_rhs, bool internal_piv, bool internal_matrix, int pass_width = 0, bool oot_diagnostics = true, int A_static_stride = dynamic_stride, int piv_static_stride = dynamic_stride, int rhs_static_stride = dynamic_stride>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool	solve_inplace_multirhs (T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT y, const int rhs_stride, const std::size_t xcol_stride, int &oot_count) noexcept
	factorize + substitute_inplace_multirhs in one call: y holds the nrhs unpermuted right-hand-side columns on entry and the nrhs solutions on exit.
template<int nrhs, bool internal_rhs, bool internal_piv, bool internal_matrix, int pass_width = 0, int A_static_stride = dynamic_stride, int piv_static_stride = dynamic_stride, int rhs_static_stride = dynamic_stride>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool	solve_inplace_multirhs (T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT y, const int rhs_stride, const std::size_t xcol_stride) noexcept
	Diagnostics-free solve_inplace_multirhs overload: no out-of-tile out-parameter at all.

Static Public Attributes
static constexpr int	tile_size = Config.tile_size
	tile size (int)
static constexpr Schedule	schedule
	elimination schedule (RightLooking or LeftLooking)
static constexpr T	oot_pivot_threshold = Config.oot_pivot_threshold
	Acceptable-pivot threshold of the out-of-tile search, read once from the configuration (see TiledLUppConfig::oot_pivot_threshold).
static constexpr T	relative_pivot_threshold = Config.relative_pivot_threshold
	Relative threshold of the in-tile pivot choice, read once from the configuration (see TiledLUppConfig::relative_pivot_threshold).
static constexpr T	singular_floor = Config.singular_floor
	Singularity floor of the out-of-tile recovery, read once from the configuration (see TiledLUppConfig::singular_floor).
static constexpr bool	transposed = Config.layout == MatrixLayout::ColMajor
	Whether the solve uses the transpose of the factored matrix. The solver factors S, the stored matrix read row-major, so that the tiles are read in address order whatever the layout. Under the row-major layout S = A and the solve is S x = b. Under the column-major layout S = A^T and the solve is S^T x = b (see the file documentation).
static constexpr bool	physical = Config.row_interchange == RowInterchange::Physical
	Whether the rows are interchanged physically, the whole matrix a single tile (see TiledLUppConfig::row_interchange).
static constexpr int	full_tiles = N / tile_size
static constexpr int	last_tile_tail = N - full_tiles * tile_size
	Extent of the last, partial tile: 0 when N is a multiple of tile_size.
static constexpr int	num_tiles = full_tiles + (last_tile_tail > 0 ? 1 : 0)
	Tiles per dimension, the partial one included.

Detailed Description

template<typename T, int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>
struct tdls::TiledLUppSolverStatic< T, N, Config >

Tiled dense LU factorization with logical partial pivoting and out-of-tile pivot recovery, solving one NxN system per call.

All entry points are static, host- and device-callable, and take raw pointers pre-offset by the caller plus one runtime stride per array (see the file documentation for the addressing convention). Entry points:

• factorize: A := P*L*U in place, pivots out (RL or LL)
• substitute: x := U^-1 L^-1 P b (b and x distinct)
• substitute_canonical: idem with b = e_col (tangent-operator columns)
• substitute_inplace: idem with b == x (cycle-decomposition permute)
• solve: factorize + substitute
• solve_inplace: factorize with the forward pass folded in

These are the row-major formulas. Under the column-major layout every substitution computes x := P^T L^-T U^-T b instead (see the Layout paragraph of the file documentation).

Each substitution and solve entry point has a _multirhs twin taking nrhs right-hand-side columns per call (substitute_multirhs, substitute_canonical_multirhs, substitute_inplace_multirhs, solve_multirhs, solve_inplace_multirhs). The columns are solved in passes of pass_width columns (0 = one single pass); every L/U tile is loaded once per pass, and per-column results match nrhs single-column calls bitwise whatever the cutting. A compiler that fuses multiplications and additions differently in the two code paths (GCC with -march=native does) changes the last bits. -ffp-contract=off (–fmad=false for nvcc) keeps them identical. The same holds for every bitwise identity between two code paths stated below. nrhs is a template parameter here and a runtime argument of the runtime solver, as the dimension itself; pass_width, a scheduling knob rather than a dimension, is a template parameter in both. It trails with a default of 0 here, so the common case needs no cutting argument at all. nrhs = 1 makes each twin collapse to its single-column form.

factorize/substitute stay separate so a factorization can be reused across several right-hand sides. Substitution is schedule-independent: RL and LL produce the same L\U layout.

The residency template booleans (internal_rhs / internal_piv / internal_matrix) deliberately have no default: they describe what the passed buffers ARE, so each call site must state them explicitly. A silent default could mismatch the actual argument and corrupt results without any diagnostic.

Template Parameters

T	scalar type (float, double or long double)
N	system dimension (N >= 1)
Config	compile-time knobs, passed as a constexpr value: tile size, schedule, pivoting thresholds, unroll policy; see TiledLUppConfig

Member Function Documentation

backward_inplace_multirhs()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int nrhs, bool internal_rhs, bool internal_piv, bool internal_matrix, int pass_width = 0, int A_static_stride = dynamic_stride, int piv_static_stride = dynamic_stride, int rhs_static_stride = dynamic_stride>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::backward_inplace_multirhs ( const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride )

inlinestaticconstexprnoexcept

Second of the two stages of substitute_inplace_multirhs, in passes of pass_width columns: x := U^-1 x under the row-major layout, x := P^T L^-T x under the column-major one.

Template Parameters

nrhs	total number of right-hand-side columns
internal_rhs	residency of x
internal_piv	residency of piv
internal_matrix	residency of A
pass_width	columns per pass, as in substitute_inplace_multirhs
A_static_stride	compile-time stride of A, used instead of A_stride; tdls::dynamic_stride (the default) reads A_stride
piv_static_stride	compile-time stride of piv, used instead of piv_stride; tdls::dynamic_stride (the default) reads piv_stride
rhs_static_stride	compile-time stride of x, used instead of rhs_stride; tdls::dynamic_stride (the default) reads rhs_stride

Parameters

[in]	A	factored matrix produced by factorize
[in]	A_stride	element stride of A (external mode)
[in]	piv	permutation produced by factorize
[in]	piv_stride	element stride of piv (external mode)
[in,out]	x	the intermediate columns on entry, the nrhs solutions on exit
[in]	rhs_stride	element stride of x (external mode)
[in]	xcol_stride	element stride between columns of x (external mode)

backward_inplace_multirhs_pass()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int pass_width, bool internal_rhs, bool internal_piv, bool internal_matrix>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::backward_inplace_multirhs_pass ( const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride )

inlinestaticconstexprnoexcept

Backward stage of one pass of substitute_inplace_multirhs: the sweep with U under the row-major layout (x := U^-1 x), the sweep with L^T then the inverse permutation under the column-major one (x := P^T L^-T x). Internal engine of backward_inplace_multirhs.

Template Parameters

pass_width	number of columns of this pass
internal_rhs	residency of x
internal_piv	residency of piv
internal_matrix	residency of A

Parameters

[in]	A	factored matrix produced by factorize
[in]	A_stride	element stride of A (external mode)
[in]	piv	permutation produced by factorize
[in]	piv_stride	element stride of piv (external mode)
[in,out]	x	pass_width columns
[in]	rhs_stride	element stride of x (external mode)
[in]	xcol_stride	element stride between columns of x (external mode)

bwd_only()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int pass_width, bool internal_rhs, bool internal_piv, bool internal_matrix>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::bwd_only ( const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride )

inlinestaticconstexprnoexcept

Backward pass alone, used by fwd_bwd and by solve_inplace (whose forward pass happens inside the factorization).

Template Parameters

pass_width	number of columns processed together
internal_rhs	residency of x
internal_piv	residency of piv
internal_matrix	residency of A

Parameters

[in]	A	factored matrix (caller-pre-offset)
[in]	A_stride	element stride of A (external mode)
[in]	piv	permutation from factorize
[in]	piv_stride	element stride of piv (external mode)
[in,out]	x	solution column(s)
[in]	rhs_stride	element stride of x (external mode)
[in]	xcol_stride	element stride between columns of x

bwd_pull_one()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int k_extent, int m_extent, int pass_width, bool internal_rhs, bool internal_piv, bool internal_matrix>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::bwd_pull_one ( const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride, const int k0, const int m0 )

inlinestaticconstexprnoexcept

Backward pull: subtract U(k,m) * x_m from the x_k segment, for pass_width columns at once.

Template Parameters

k_extent	extent of the updated segment's tile
m_extent	extent of the trailing segment's tile
pass_width	number of columns processed together
internal_rhs	residency of x
internal_piv	residency of piv
internal_matrix	residency of A

Parameters

[in]	A	factored matrix (caller-pre-offset)
[in]	A_stride	element stride of A (external mode)
[in]	piv	permutation from factorize
[in]	piv_stride	element stride of piv (external mode)
[in,out]	x	solution column(s)
[in]	rhs_stride	element stride of x (external mode)
[in]	xcol_stride	element stride between columns of x
[in]	k0	first global row of the updated segment
[in]	m0	first global row of the trailing segment

bwd_step()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int k_extent, int pass_width, bool internal_rhs, bool internal_piv, bool internal_matrix>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::bwd_step ( const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride, const int k )

inlinestaticconstexprnoexcept

Backward step: pull the trailing contributions, then upper-solve the diagonal tile's segment.

Template Parameters

k_extent	extent of the diagonal tile of the step
pass_width	number of columns processed together
internal_rhs	residency of x
internal_piv	residency of piv
internal_matrix	residency of A

Parameters

[in]	A	factored matrix (caller-pre-offset)
[in]	A_stride	element stride of A (external mode)
[in]	piv	permutation from factorize
[in]	piv_stride	element stride of piv (external mode)
[in,out]	x	solution column(s)
[in]	rhs_stride	element stride of x (external mode)
[in]	xcol_stride	element stride between columns of x
[in]	k	step index (k0 = k*tile_size)

factor_diag_column()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int k_extent, bool internal_piv, bool internal_matrix, bool oot_diagnostics, bool fuse_rhs, bool internal_rhs>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool tdls::TiledLUppSolverStatic< T, N, Config >::factor_diag_column ( T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride, const int k0, T *TDLS_RESTRICT tile, int & oot_count, const int c, T *TDLS_RESTRICT y, const int rhs_stride )

inlinestaticnodiscardconstexprnoexcept

One column step of the diagonal-tile factorization: pivot search (in-tile, then out-of-tile recovery), permutation update, row swap or cross-tile pull, column elimination.

tile holds the (LL: prior-corrected) KExKE diagonal tile; physical row swaps happen in the register tile only.

Template Parameters

k_extent	extent of the diagonal tile
internal_piv	residency of piv
internal_matrix	residency of A
oot_diagnostics	compile the out-of-tile counter in or out
fuse_rhs	apply the pivot swaps to the fused RHS y (row-major layout; the column-major one swaps nothing)
internal_rhs	residency of y

Parameters

[in,out]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A (external mode)
[in,out]	piv	permutation (logical -> physical row)
[in]	piv_stride	element stride of piv (external mode)
[in]	k0	first global row/column of the tile
[in,out]	tile	register-resident diagonal tile
[in,out]	oot_count	out-of-tile search counter (oot_diagnostics)
[in]	c	tile column to factor
[in,out]	y	fused right-hand side (fuse_rhs only)
[in]	rhs_stride	element stride of y (external mode)

Returns

false when the matrix is singular at this column.

factor_diag_tile()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int k_extent, bool internal_piv, bool internal_matrix, bool oot_diagnostics, bool fuse_rhs = false, bool internal_rhs = true>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool tdls::TiledLUppSolverStatic< T, N, Config >::factor_diag_tile ( T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride, const int k0, T *TDLS_RESTRICT tile, int & oot_count, T *TDLS_RESTRICT y = nullptr, const int rhs_stride = 1 )

inlinestaticnodiscardconstexprnoexcept

Factor the KExKE diagonal tile in registers, with out-of-tile pivot recovery (drives the per-column loop of factor_diag_column).

Template Parameters

k_extent	extent of the diagonal tile
internal_piv	residency of piv
internal_matrix	residency of A
oot_diagnostics	compile the out-of-tile counter in or out
fuse_rhs	apply the pivot swaps to the fused RHS y (row-major layout; the column-major one swaps nothing)
internal_rhs	residency of y

Parameters

[in,out]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A (external mode)
[in,out]	piv	permutation (logical -> physical row)
[in]	piv_stride	element stride of piv (external mode)
[in]	k0	first global row/column of the tile
[in,out]	tile	register-resident diagonal tile, loaded (and, LL: prior-corrected) by the caller
[in,out]	oot_count	out-of-tile search counter (oot_diagnostics)
[in,out]	y	fused right-hand side (fuse_rhs only)
[in]	rhs_stride	element stride of y (external mode)

Returns

false on a singular matrix.

factorize() [1/2]

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<bool internal_piv, bool internal_matrix, int A_static_stride = dynamic_stride, int piv_static_stride = dynamic_stride>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool tdls::TiledLUppSolverStatic< T, N, Config >::factorize ( T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride )

inlinestaticnodiscardconstexprnoexcept

Diagnostics-free factorize overload: no out-of-tile out-parameter at all.

Template Parameters

internal_piv	residency of piv
internal_matrix	residency of A
A_static_stride	compile-time stride of A, used instead of A_stride; tdls::dynamic_stride (the default) reads A_stride
piv_static_stride	compile-time stride of piv, used instead of piv_stride; tdls::dynamic_stride (the default) reads piv_stride

Parameters

[in,out]	A	matrix, pre-offset by the caller
[in]	A_stride	element stride of A (external mode)
[out]	piv	permutation storage (always caller-provided)
[in]	piv_stride	element stride of piv (external mode)

Returns

false on a singular matrix.

factorize() [2/2]

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<bool internal_piv, bool internal_matrix, bool oot_diagnostics = true, bool fuse_rhs = false, bool internal_rhs = true, int A_static_stride = dynamic_stride, int piv_static_stride = dynamic_stride, int rhs_static_stride = dynamic_stride>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool tdls::TiledLUppSolverStatic< T, N, Config >::factorize ( T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride, int & oot_count, T *TDLS_RESTRICT y = nullptr, const int rhs_stride = 1 )

inlinestaticnodiscardconstexprnoexcept

Factor A := P*L*U in place.

A becomes L\U (unit lower L below the diagonal, U above including it, the diagonal holding the pivot RECIPROCALS), under logical row permutation piv (piv[i] = physical row holding logical row i). Under physical row interchanges (Config.row_interchange), L\U is stored in the pivoted row order and piv[j] is the row exchanged with row j at step j.

Template Parameters

internal_piv	true: piv is a caller-local int[N], stride ignored; false: piv is remote, indexed with piv_stride
internal_matrix	true: A is a caller-local T[N*N], stride ignored; false: A is remote, indexed with A_stride
oot_diagnostics	when false, the out-of-tile diagnostics are compiled out entirely (no counter register, no increments, oot_count untouched); use the overload without the out-parameter

Parameters

[in,out]	A	matrix, pre-offset by the caller
[in]	A_stride	element stride of A (external mode)
[out]	piv	permutation storage (always caller-provided)
[in]	piv_stride	element stride of piv (external mode)

Template Parameters

fuse_rhs	internal hook of solve_inplace: folds the forward substitution of y into the factorization
internal_rhs	residency of the fused y (meaningful with fuse_rhs only)
A_static_stride	compile-time stride of A, used instead of A_stride; tdls::dynamic_stride (the default) reads A_stride
piv_static_stride	compile-time stride of piv, used instead of piv_stride; tdls::dynamic_stride (the default) reads piv_stride
rhs_static_stride	compile-time stride of the fused y, used instead of rhs_stride; tdls::dynamic_stride (the default) reads rhs_stride

Parameters

[out]	oot_count	number of columns whose best in-tile pivot fell below oot_pivot_threshold
[in,out]	y	fused right-hand side (fuse_rhs only)
[in]	rhs_stride	element stride of y (external mode)

Returns

false on a singular matrix.

forward_inplace_multirhs()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int nrhs, bool internal_rhs, bool internal_piv, bool internal_matrix, int pass_width = 0, int A_static_stride = dynamic_stride, int piv_static_stride = dynamic_stride, int rhs_static_stride = dynamic_stride>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::forward_inplace_multirhs ( const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride )

inlinestaticconstexprnoexcept

First of the two stages of substitute_inplace_multirhs, in passes of pass_width columns: x := L^-1 P x under the row-major layout, x := U^-T x under the column-major one. The caller may read or update the intermediate columns, then backward_inplace_multirhs completes the solve; the two stages in a row match substitute_inplace_multirhs bitwise.

Template Parameters

nrhs	total number of right-hand-side columns
internal_rhs	residency of x
internal_piv	residency of piv
internal_matrix	residency of A
pass_width	columns per pass, as in substitute_inplace_multirhs
A_static_stride	compile-time stride of A, used instead of A_stride; tdls::dynamic_stride (the default) reads A_stride
piv_static_stride	compile-time stride of piv, used instead of piv_stride; tdls::dynamic_stride (the default) reads piv_stride
rhs_static_stride	compile-time stride of x, used instead of rhs_stride; tdls::dynamic_stride (the default) reads rhs_stride

Parameters

[in]	A	factored matrix produced by factorize
[in]	A_stride	element stride of A (external mode)
[in]	piv	permutation produced by factorize
[in]	piv_stride	element stride of piv (external mode)
[in,out]	x	nrhs right-hand-side columns on entry, the intermediate columns on exit
[in]	rhs_stride	element stride of x (external mode)
[in]	xcol_stride	element stride between columns of x (external mode)

forward_inplace_multirhs_pass()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int pass_width, bool internal_rhs, bool internal_piv, bool internal_matrix>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::forward_inplace_multirhs_pass ( const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride )

inlinestaticconstexprnoexcept

Forward stage of one pass of substitute_inplace_multirhs: the permutation and the sweep with L under the row-major layout (x := L^-1 P x), the sweep with U^T under the column-major one (x := U^-T x). Internal engine of forward_inplace_multirhs.

Template Parameters

pass_width	number of columns of this pass
internal_rhs	residency of x
internal_piv	residency of piv
internal_matrix	residency of A

Parameters

[in]	A	factored matrix produced by factorize
[in]	A_stride	element stride of A (external mode)
[in]	piv	permutation produced by factorize
[in]	piv_stride	element stride of piv (external mode)
[in,out]	x	pass_width columns
[in]	rhs_stride	element stride of x (external mode)
[in]	xcol_stride	element stride between columns of x (external mode)

fwd_bwd()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int pass_width, bool internal_rhs, bool internal_piv, bool internal_matrix>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::fwd_bwd ( const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride )

inlinestaticconstexprnoexcept

Triangular solves on already-permuted column(s) x. pass_width columns are processed per tile visit, so every L/U tile is loaded once for the whole block instead of once per column (pass_width = 1 is the single-RHS case).

Template Parameters

pass_width	number of columns processed together
internal_rhs	residency of x
internal_piv	residency of piv
internal_matrix	residency of A

Parameters

[in]	A	factored matrix (caller-pre-offset)
[in]	A_stride	element stride of A (external mode)
[in]	piv	permutation from factorize
[in]	piv_stride	element stride of piv (external mode)
[in,out]	x	solution column(s)
[in]	rhs_stride	element stride of x (external mode)
[in]	xcol_stride	element stride between columns of x

fwd_push_one()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int k_extent, int m_extent, int pass_width, bool internal_rhs, bool internal_piv, bool internal_matrix>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::fwd_push_one ( const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride, const int k0, const int m0 )

inlinestaticconstexprnoexcept

Forward push: subtract L(m,k) * x_k from the x_m segment, for pass_width columns at once.

Template Parameters

k_extent	extent of the solved segment's tile
m_extent	extent of the target segment's tile
pass_width	number of columns processed together
internal_rhs	residency of x
internal_piv	residency of piv
internal_matrix	residency of A

Parameters

[in]	A	factored matrix (caller-pre-offset)
[in]	A_stride	element stride of A (external mode)
[in]	piv	permutation from factorize
[in]	piv_stride	element stride of piv (external mode)
[in,out]	x	solution column(s)
[in]	rhs_stride	element stride of x (external mode)
[in]	xcol_stride	element stride between columns of x
[in]	k0	first global row of the solved segment
[in]	m0	first global row of the target segment

fwd_step()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int k_extent, int pass_width, bool internal_rhs, bool internal_piv, bool internal_matrix>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::fwd_step ( const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride, const int k )

inlinestaticconstexprnoexcept

Forward step: unit-lower solve the diagonal tile's segment, then push it into the tiles below.

Template Parameters

k_extent	extent of the diagonal tile of the step
pass_width	number of columns processed together
internal_rhs	residency of x
internal_piv	residency of piv
internal_matrix	residency of A

Parameters

[in]	A	factored matrix (caller-pre-offset)
[in]	A_stride	element stride of A (external mode)
[in]	piv	permutation from factorize
[in]	piv_stride	element stride of piv (external mode)
[in,out]	x	solution column(s)
[in]	rhs_stride	element stride of x (external mode)
[in]	xcol_stride	element stride between columns of x
[in]	k	step index (k0 = k*tile_size)

ll_correct_tile()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int row_extent, int col_extent, bool internal_piv, bool internal_matrix>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::ll_correct_tile ( const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, const int row0, const int col0, const int k0, T *TDLS_RESTRICT t )

inlinestaticconstexprnoexcept

LL: t (RExCE, rows row0.., cols col0..) -= sum over prior tiles bj < k0/tile_size of L(row0.., bj) * U(bj, col0..). All prior tiles are full.

Template Parameters

row_extent	row extent of the corrected tile
col_extent	column extent of the corrected tile
internal_piv	residency of piv
internal_matrix	residency of A

Parameters

[in]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A (external mode)
[in]	piv	permutation (logical -> physical row)
[in]	piv_stride	element stride of piv (external mode)
[in]	row0	first global row of the corrected tile
[in]	col0	first global column of the corrected tile
[in]	k0	first row/column of the current step
[in,out]	t	register tile being corrected

ll_step()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int k_extent, bool internal_piv, bool internal_matrix, bool oot_diagnostics, bool fuse_rhs = false, bool internal_rhs = true>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool tdls::TiledLUppSolverStatic< T, N, Config >::ll_step ( T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride, const int k, int & oot_count, T *TDLS_RESTRICT y = nullptr, const int rhs_stride = 1 )

inlinestaticnodiscardconstexprnoexcept

LL: one full factorization step (correct + factor the diagonal tile, then its L and U panels).

Template Parameters

k_extent	extent of the diagonal tile of this step
internal_piv	residency of piv
internal_matrix	residency of A
oot_diagnostics	compile the out-of-tile counter in or out
fuse_rhs	apply the step to the fused RHS y (solve_inplace)
internal_rhs	residency of y

Parameters

[in,out]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A (external mode)
[in,out]	piv	permutation (logical -> physical row)
[in]	piv_stride	element stride of piv (external mode)
[in]	k	step index (k0 = k*tile_size)
[in,out]	oot_count	out-of-tile search counter (oot_diagnostics)
[in,out]	y	fused right-hand side (fuse_rhs only)
[in]	rhs_stride	element stride of y (external mode)

Returns

false on a singular matrix.

ll_update_below_one()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int k_extent, int i_extent, bool internal_piv, bool internal_matrix, bool fuse_rhs = false, bool internal_rhs = true>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::ll_update_below_one ( T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, const T *TDLS_RESTRICT tile, const int k0, const int i0, T *TDLS_RESTRICT y = nullptr, const int rhs_stride = 1 )

inlinestaticconstexprnoexcept

LL: correct + TRSM one L-panel tile below the diagonal, with the optional fused forward-substitution push.

Template Parameters

k_extent	extent of the diagonal tile
i_extent	row extent of the updated tile
internal_piv	residency of piv
internal_matrix	residency of A
fuse_rhs	push the fused RHS y along (solve_inplace)
internal_rhs	residency of y

Parameters

[in,out]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A (external mode)
[in]	piv	permutation (logical -> physical row)
[in]	piv_stride	element stride of piv (external mode)
[in]	tile	factored diagonal tile
[in]	k0	first global row/column of the step
[in]	i0	first global row of the updated tile
[in,out]	y	fused right-hand side (fuse_rhs only)
[in]	rhs_stride	element stride of y (external mode)

ll_update_right_one()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int k_extent, int j_extent, bool internal_piv, bool internal_matrix>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::ll_update_right_one ( T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, const T *TDLS_RESTRICT tile, const int k0, const int j0 )

inlinestaticconstexprnoexcept

LL: correct + TRSM one U-panel tile right of the diagonal.

Template Parameters

k_extent	extent of the diagonal tile
j_extent	column extent of the updated tile
internal_piv	residency of piv
internal_matrix	residency of A

Parameters

[in,out]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A (external mode)
[in]	piv	permutation (logical -> physical row)
[in]	piv_stride	element stride of piv (external mode)
[in]	tile	factored diagonal tile
[in]	k0	first global row/column of the step
[in]	j0	first global column of the updated tile

ll_update_right_push_one()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int k_extent, int j_extent, bool internal_piv, bool internal_matrix, bool internal_rhs>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::ll_update_right_push_one ( T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, const T *TDLS_RESTRICT tile, const int k0, const int j0, T *TDLS_RESTRICT y, const int rhs_stride )

inlinestaticconstexprnoexcept

LL, column-major layout, solve_inplace: ll_update_right_one, then the fused U^T forward push. B is then the final U(k,j) tile: the solved z_k segment is pushed into the y_j segment while the tile sits in registers.

Template Parameters

k_extent	extent of the diagonal tile
j_extent	column extent of the updated tile
internal_piv	residency of piv
internal_matrix	residency of A
internal_rhs	residency of y

Parameters

[in,out]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A (external mode)
[in]	piv	permutation (logical -> physical row)
[in]	piv_stride	element stride of piv (external mode)
[in]	tile	factored diagonal tile
[in]	k0	first global row/column of the step
[in]	j0	first global column of the updated tile
[in,out]	y	fused right-hand side
[in]	rhs_stride	element stride of y (external mode)

load_tile()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int row_extent, int col_extent, bool internal_matrix>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::load_tile ( const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT prow, const int col0, T *TDLS_RESTRICT t )

inlinestaticconstexprnoexcept

Load an RxC tile through a cached physical-row segment.

Template Parameters

row_extent	tile row extent
col_extent	tile column extent
internal_matrix	residency of A

Parameters

[in]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A (external mode)
[in]	prow	physical rows of the tile
[in]	col0	first global column of the tile
[out]	t	destination register tile

load_tile_piv()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int row_extent, int col_extent, bool internal_piv, bool internal_matrix>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::load_tile_piv ( const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, const int row0, const int col0, T *TDLS_RESTRICT t )

inlinestaticconstexprnoexcept

Load an RxC tile, reading the permutation inline (one read per row).

Template Parameters

row_extent	tile row extent
col_extent	tile column extent
internal_piv	residency of piv
internal_matrix	residency of A

Parameters

[in]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A (external mode)
[in]	piv	permutation (logical -> physical row)
[in]	piv_stride	element stride of piv (external mode)
[in]	row0	first global (logical) row of the tile
[in]	col0	first global column of the tile
[out]	t	destination register tile

load_tile_piv_lower()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int row_extent, bool internal_piv, bool internal_matrix>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::load_tile_piv_lower ( const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, const int row0, const int col0, T *TDLS_RESTRICT t )

inlinestaticconstexprnoexcept

Triangular variant of the diagonal-tile load for the forward substitution (the backward one under the column-major layout): only the strict lower triangle of L\U is read.

The untouched slots of the register tile are never read: compile-time loop bounds, no branches, results bitwise identical.

Template Parameters

row_extent	tile extent
internal_piv	residency of piv
internal_matrix	residency of A

Parameters

[in]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A (external mode)
[in]	piv	permutation (logical -> physical row)
[in]	piv_stride	element stride of piv (external mode)
[in]	row0	first global (logical) row of the tile
[in]	col0	first global column of the tile
[out]	t	destination register tile

load_tile_piv_upper()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int row_extent, bool internal_piv, bool internal_matrix>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::load_tile_piv_upper ( const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, const int row0, const int col0, T *TDLS_RESTRICT t )

inlinestaticconstexprnoexcept

Triangular variant of the diagonal-tile load for the backward substitution (the forward one under the column-major layout): only the upper triangle including the diagonal is read.

Template Parameters

row_extent	tile extent
internal_piv	residency of piv
internal_matrix	residency of A

Parameters

[in]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A (external mode)
[in]	piv	permutation (logical -> physical row)
[in]	piv_stride	element stride of piv (external mode)
[in]	row0	first global (logical) row of the tile
[in]	col0	first global column of the tile
[out]	t	destination register tile

permute_cycles()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int pass_width, bool internal_rhs, bool internal_piv>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_ALL constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::permute_cycles ( const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride )

inlinestaticconstexprnoexcept

x := P x in place for pass_width columns (row i receives row piv[i]), by the cycle decomposition of substitute_inplace, each move carrying the pass_width column entries of its row: the permutation of substitute_inplace_multirhs_pass and forward_inplace_multirhs, the inverse of unpermute_cycles. Forced inline on CPU too (TDLS_FORCE_INLINE_ALL): an out-of-line call costs 20 to 27 % under gcc -O2, and inlined it leaves the code of substitute_inplace_multirhs as it was when these lines sat in it.

Template Parameters

pass_width	number of columns of x
internal_rhs	residency of x
internal_piv	residency of piv

Parameters

[in]	piv	permutation from factorize
[in]	piv_stride	element stride of piv (external mode)
[in,out]	x	right-hand side column(s)
[in]	rhs_stride	element stride of x (external mode)
[in]	xcol_stride	element stride between columns of x

physical_backward()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int w, bool internal_rhs, bool internal_piv, bool internal_matrix>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::physical_backward ( const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride )

inlinestaticconstexprnoexcept

Second stage of the physical substitution of w columns: x := U^-1 x under the row-major layout, x := P^T L^-T x under the column-major one (the rows of L^T, then the interchanges in reverse order).

Template Parameters

w	number of columns
internal_rhs	residency of x
internal_piv	residency of piv
internal_matrix	residency of A

Parameters

[in]	A	factors of physical_factorize
[in]	A_stride	element stride of A (external mode)
[in]	piv	interchanges
[in]	piv_stride	element stride of piv (external mode)
[in,out]	x	the columns
[in]	rhs_stride	element stride of x (external mode)
[in]	xcol_stride	element stride between columns of x

physical_factorize()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<bool internal_piv, bool internal_matrix>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool tdls::TiledLUppSolverStatic< T, N, Config >::physical_factorize ( T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride, int & weak )

inlinestaticnodiscardconstexprnoexcept

Physical factorization: P S = L U in place, L below the diagonal and U above it in the pivoted row order, 1 / pivot on the diagonal; piv[j] the row exchanged with row j at step j.

Template Parameters

internal_piv	residency of piv
internal_matrix	residency of A

Parameters

[in,out]	A	matrix on entry, its factors on exit
[in]	A_stride	element stride of A (external mode)
[out]	piv	interchanges
[in]	piv_stride	element stride of piv (external mode)
[out]	weak	number of columns whose best candidate fell below oot_pivot_threshold

Returns

false on a singular matrix

physical_factorize_checked()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<bool internal_piv, bool internal_matrix>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr int tdls::TiledLUppSolverStatic< T, N, Config >::physical_factorize_checked ( T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride, const T u )

inlinestaticnodiscardconstexprnoexcept

Physical factorization with its multipliers bounded, the check fused into the elimination (the condensation of ArrowheadLUpp under RowInterchange::Physical): as physical_factorize, and every multiplier of L checked against 1 / u.

Template Parameters

internal_piv	residency of piv
internal_matrix	residency of A

Parameters

[in,out]	A	matrix on entry, its factors on exit
[in]	A_stride	element stride of A (external mode)
[out]	piv	interchanges
[in]	piv_stride	element stride of piv (external mode)
[in]	u	the multipliers must not exceed 1 / u

Returns

0 when every multiplier passes the check, 1 when one fails it, 2 on a singular matrix

physical_forward()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int w, bool internal_rhs, bool internal_piv, bool internal_matrix>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::physical_forward ( const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride )

inlinestaticconstexprnoexcept

First stage of the physical substitution of w columns: x := L^-1 P x under the row-major layout (the interchanges in their order, then the columns of L), x := U^-T x under the column-major one.

Template Parameters

w	number of columns
internal_rhs	residency of x
internal_piv	residency of piv
internal_matrix	residency of A

Parameters

[in]	A	factors of physical_factorize
[in]	A_stride	element stride of A (external mode)
[in]	piv	interchanges
[in]	piv_stride	element stride of piv (external mode)
[in,out]	x	the columns
[in]	rhs_stride	element stride of x (external mode)
[in]	xcol_stride	element stride between columns of x

physical_interchange_local()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int n>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::physical_interchange_local ( T *TDLS_RESTRICT X, const int j, const int r, const int cs, const int rs )

inlinestaticconstexprnoexcept

Interchanges rows j and r > j of the n columns of X, entry (i, c) at i * rs + c * cs, X a caller-local array (physical scheme): a test of every unrolled row index against r. Written with the strides as arguments, as the former register path of ArrowheadLUpp wrote it: through the addressing macros, nvcc dispatches the interchanges of several passes through more jump tables, 4 % slower on average on an RTX 5080.

Template Parameters

n	number of columns

Parameters

[in,out]	X	the columns
[in]	j	first row
[in]	r	second row
[in]	cs	column stride of X
[in]	rs	row stride of X

physical_interchange_rhs()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int w, bool internal_rhs>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::physical_interchange_rhs ( T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride, const int j, const int r )

inlinestaticconstexprnoexcept

Interchanges rows j and r > j of the w columns of x (physical scheme): a test of every unrolled row index against r.

Template Parameters

w	number of columns
internal_rhs	residency of x

Parameters

[in,out]	x	the columns
[in]	rhs_stride	element stride of x (external mode)
[in]	xcol_stride	element stride between columns of x
[in]	j	first row
[in]	r	second row

physical_multirhs()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int nrhs, int pass_width, int stages, bool internal_rhs, bool internal_piv, bool internal_matrix>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::physical_multirhs ( const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride )

inlinestaticconstexprnoexcept

Physical substitution of nrhs columns in passes of pass_width columns: the first stage, the second, or both.

Template Parameters

nrhs	number of columns
pass_width	columns per pass (0: one pass)
stages	1 the first stage, 2 the second, 3 both
internal_rhs	residency of x
internal_piv	residency of piv
internal_matrix	residency of A

Parameters

[in]	A	factors of physical_factorize
[in]	A_stride	element stride of A (external mode)
[in]	piv	interchanges
[in]	piv_stride	element stride of piv (external mode)
[in,out]	x	the columns
[in]	rhs_stride	element stride of x (external mode)
[in]	xcol_stride	element stride between columns of x

physical_permute()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int w, bool internal_rhs, bool internal_piv, bool reverse>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::physical_permute ( const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride )

inlinestaticconstexprnoexcept

Applies the interchanges to w columns of x (physical scheme): in their order (P x), or in reverse order (P^T x).

Template Parameters

w	number of columns
internal_rhs	residency of x
internal_piv	residency of piv
reverse	P^T instead of P

Parameters

[in]	piv	interchanges
[in]	piv_stride	element stride of piv (external mode)
[in,out]	x	the columns
[in]	rhs_stride	element stride of x (external mode)
[in]	xcol_stride	element stride between columns of x

physical_step()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<bool internal_piv, bool internal_matrix, bool check>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool tdls::TiledLUppSolverStatic< T, N, Config >::physical_step ( T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride, const int j, const T u, bool & singular, int & weak )

inlinestaticnodiscardconstexprnoexcept

Step j of the physical factorization: the pivot of column j by the rule of a single tile (threshold pivoting, the singularity floor), the interchange of the whole rows, the elimination below the pivot. Branch-free over the steps: a column without a candidate above singular_floor sets singular and the elimination goes on with a zero reciprocal, so that the unrolled steps run as straight-line code.

Template Parameters

internal_piv	residency of piv
internal_matrix	residency of A
check	bound the multipliers by 1 / u

Parameters

[in,out]	A	matrix, factored up to column j
[in]	A_stride	element stride of A (external mode)
[out]	piv	interchanges
[in]	piv_stride	element stride of piv (external mode)
[in]	j	column
[in]	u	bound of the check (check only)
[in,out]	singular	set when the column has no candidate above singular_floor
[in,out]	weak	incremented when the best candidate of the column falls below oot_pivot_threshold (the out-of-tile count of Logical over one tile)

Returns

false when a multiplier of the column exceeds 1 / u (check)

rl_schur_one()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int k_extent, int i_extent, int j_extent, bool internal_matrix>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::rl_schur_one ( T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT pk, const int *TDLS_RESTRICT pi, const T *TDLS_RESTRICT Aik, const int j0 )

inlinestaticconstexprnoexcept

RL: one Schur-complement tile update, Aij -= Aik * Akj, streaming the factored Akj row by row from remote memory (it is not worth a third register tile).

Template Parameters

k_extent	inner extent (current step)
i_extent	row extent of the updated tile
j_extent	column extent of the updated tile
internal_matrix	residency of A

Parameters

[in,out]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A (external mode)
[in]	pk	physical rows of the factored block row
[in]	pi	physical rows of the updated block row
[in]	Aik	register L panel of the updated block row
[in]	j0	first global column of the updated tile

rl_step()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int k_extent, bool internal_piv, bool internal_matrix, bool oot_diagnostics, bool fuse_rhs = false, bool internal_rhs = true>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool tdls::TiledLUppSolverStatic< T, N, Config >::rl_step ( T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride, const int k, int & oot_count, T *TDLS_RESTRICT y = nullptr, const int rhs_stride = 1 )

inlinestaticnodiscardconstexprnoexcept

RL: one full factorization step (diagonal tile + trailing updates).

Template Parameters

k_extent	extent of the diagonal tile of this step
internal_piv	residency of piv
internal_matrix	residency of A
oot_diagnostics	compile the out-of-tile counter in or out
fuse_rhs	apply the step to the fused RHS y (solve_inplace)
internal_rhs	residency of y

Parameters

[in,out]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A (external mode)
[in,out]	piv	permutation (logical -> physical row)
[in]	piv_stride	element stride of piv (external mode)
[in]	k	step index (k0 = k*tile_size)
[in,out]	oot_count	out-of-tile search counter (oot_diagnostics)
[in,out]	y	fused right-hand side (fuse_rhs only)
[in]	rhs_stride	element stride of y (external mode)

Returns

false on a singular matrix.

rl_trsm_right_one()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int k_extent, int j_extent, bool internal_matrix>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::rl_trsm_right_one ( T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT pk, const T *TDLS_RESTRICT tile, const int j0 )

inlinestaticconstexprnoexcept

RL: one row-panel tile update, Akj := L^-1 Akj.

Template Parameters

k_extent	extent of the factored diagonal tile
j_extent	column extent of the updated tile
internal_matrix	residency of A

Parameters

[in,out]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A (external mode)
[in]	pk	physical rows of the diagonal block row
[in]	tile	factored diagonal tile
[in]	j0	first global column of the updated tile

rl_trsm_right_push_one()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int k_extent, int j_extent, bool internal_matrix, bool internal_rhs>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::rl_trsm_right_push_one ( T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT pk, const T *TDLS_RESTRICT tile, const int j0, const int k0, T *TDLS_RESTRICT y, const int rhs_stride )

inlinestaticconstexprnoexcept

RL, column-major layout, solve_inplace: rl_trsm_right_one, then the fused U^T forward push. Akj is then the final U(k,j) tile: the solved z_k segment is pushed into the y_j segment while the tile sits in registers.

Template Parameters

k_extent	extent of the factored diagonal tile
j_extent	column extent of the updated tile
internal_matrix	residency of A
internal_rhs	residency of y

Parameters

[in,out]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A (external mode)
[in]	pk	physical rows of the diagonal block row
[in]	tile	factored diagonal tile
[in]	j0	first global column of the updated tile
[in]	k0	first global row/column of the step
[in,out]	y	fused right-hand side
[in]	rhs_stride	element stride of y (external mode)

rl_update_row_one()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int k_extent, int i_extent, bool internal_piv, bool internal_matrix, bool fuse_rhs = false, bool internal_rhs = true>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::rl_update_row_one ( T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, const int *TDLS_RESTRICT pk, const T *TDLS_RESTRICT tile, const int k, const int i0, T *TDLS_RESTRICT y = nullptr, const int rhs_stride = 1 )

inlinestaticconstexprnoexcept

RL: TRSM down + Schur sweep of one row block below the diagonal, with the optional fused forward-substitution push.

Template Parameters

k_extent	extent of the diagonal tile
i_extent	row extent of the updated block row
internal_piv	residency of piv
internal_matrix	residency of A
fuse_rhs	push the fused RHS y along (solve_inplace)
internal_rhs	residency of y

Parameters

[in,out]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A (external mode)
[in]	piv	permutation (logical -> physical row)
[in]	piv_stride	element stride of piv (external mode)
[in]	pk	physical rows of the diagonal tile
[in]	tile	factored diagonal tile
[in]	k	step index (k0 = k*tile_size)
[in]	i0	first global row of the updated block row
[in,out]	y	fused right-hand side (fuse_rhs only)
[in]	rhs_stride	element stride of y (external mode)

solve() [1/2]

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<bool internal_rhs, bool internal_piv, bool internal_matrix, int A_static_stride = dynamic_stride, int piv_static_stride = dynamic_stride, int rhs_static_stride = dynamic_stride>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool tdls::TiledLUppSolverStatic< T, N, Config >::solve ( T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride, const T *TDLS_RESTRICT b, T *TDLS_RESTRICT x, const int rhs_stride )

inlinestaticnodiscardconstexprnoexcept

Diagnostics-free solve overload: no out-of-tile out-parameter at all.

Template Parameters

internal_rhs	residency of b and x
internal_piv	residency of piv
internal_matrix	residency of A
A_static_stride	compile-time stride of A, used instead of A_stride; tdls::dynamic_stride (the default) reads A_stride
piv_static_stride	compile-time stride of piv, used instead of piv_stride; tdls::dynamic_stride (the default) reads piv_stride
rhs_static_stride	compile-time stride of b and x, used instead of rhs_stride; tdls::dynamic_stride (the default) reads rhs_stride

Parameters

[in,out]	A	on entry the matrix (pre-offset by the caller), on exit its factorization
[in]	A_stride	element stride of A (external mode)
[out]	piv	permutation storage (always caller-provided)
[in]	piv_stride	element stride of piv (external mode)
[in]	b	right-hand side, in original order
[out]	x	solution
[in]	rhs_stride	element stride of b and x (external mode)

Returns

false on a singular matrix.

solve() [2/2]

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<bool internal_rhs, bool internal_piv, bool internal_matrix, bool oot_diagnostics = true, int A_static_stride = dynamic_stride, int piv_static_stride = dynamic_stride, int rhs_static_stride = dynamic_stride>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool tdls::TiledLUppSolverStatic< T, N, Config >::solve ( T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride, const T *TDLS_RESTRICT b, T *TDLS_RESTRICT x, const int rhs_stride, int & oot_count )

inlinestaticnodiscardconstexprnoexcept

factorize + substitute in one call.

Template Parameters

internal_rhs	residency of b and x
internal_piv	residency of piv
internal_matrix	residency of A
oot_diagnostics	compile the out-of-tile counter in or out
A_static_stride	compile-time stride of A, used instead of A_stride; tdls::dynamic_stride (the default) reads A_stride
piv_static_stride	compile-time stride of piv, used instead of piv_stride; tdls::dynamic_stride (the default) reads piv_stride
rhs_static_stride	compile-time stride of b and x, used instead of rhs_stride; tdls::dynamic_stride (the default) reads rhs_stride

Parameters

[in,out]	A	on entry the matrix (pre-offset by the caller), on exit its factorization (usable for further substitute* calls)
[in]	A_stride	element stride of A (external mode)
[out]	piv	permutation storage (always caller-provided)
[in]	piv_stride	element stride of piv (external mode)
[in]	b	right-hand side, in original order
[out]	x	solution
[in]	rhs_stride	element stride of b and x (external mode)
[out]	oot_count	number of columns whose best in-tile pivot fell below oot_pivot_threshold

Returns

false on a singular matrix.

solve_inplace() [1/2]

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<bool internal_rhs, bool internal_piv, bool internal_matrix, int A_static_stride = dynamic_stride, int piv_static_stride = dynamic_stride, int rhs_static_stride = dynamic_stride>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool tdls::TiledLUppSolverStatic< T, N, Config >::solve_inplace ( T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT y, const int rhs_stride )

inlinestaticnodiscardconstexprnoexcept

Diagnostics-free solve_inplace overload: no out-of-tile out-parameter at all.

Template Parameters

internal_rhs	residency of y
internal_piv	residency of piv
internal_matrix	residency of A
A_static_stride	compile-time stride of A, used instead of A_stride; tdls::dynamic_stride (the default) reads A_stride
piv_static_stride	compile-time stride of piv, used instead of piv_stride; tdls::dynamic_stride (the default) reads piv_stride
rhs_static_stride	compile-time stride of y, used instead of rhs_stride; tdls::dynamic_stride (the default) reads rhs_stride

Parameters

[in,out]	A	on entry the matrix (pre-offset by the caller), on exit its factorization
[in]	A_stride	element stride of A (external mode)
[out]	piv	permutation storage (always caller-provided)
[in]	piv_stride	element stride of piv (external mode)
[in,out]	y	right-hand side on entry, solution on exit
[in]	rhs_stride	element stride of y (external mode)

Returns

false on a singular matrix (y left partially updated).

solve_inplace() [2/2]

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<bool internal_rhs, bool internal_piv, bool internal_matrix, bool oot_diagnostics = true, int A_static_stride = dynamic_stride, int piv_static_stride = dynamic_stride, int rhs_static_stride = dynamic_stride>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool tdls::TiledLUppSolverStatic< T, N, Config >::solve_inplace ( T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT y, const int rhs_stride, int & oot_count )

inlinestaticnodiscardconstexprnoexcept

Factorization with the forward substitution folded in.

y holds the unpermuted RHS on entry and the solution on exit. Each L panel is applied to y while it sits in registers during the factorization, so the separate forward pass disappears, and all its tile reloads with it; only the backward pass remains. y follows the rows through pivoting (swap hook in factor_diag_column), which makes the result bitwise-identical to factorize + substitute_inplace when the compiler contracts multiply-adds alike in both (see the class documentation). Under the column-major layout, the U panels are applied instead (U^T z = b, each diagonal tile solved as soon as it is factored), y is not swapped, and the backward pass L^T w = z is followed by x = P^T w: the same sweep kernels as substitute_inplace, and a final permutation that only moves values, hence the same bitwise identity.

Template Parameters

internal_rhs	residency of y
internal_piv	residency of piv
internal_matrix	residency of A
oot_diagnostics	compile the out-of-tile counter in or out
A_static_stride	compile-time stride of A, used instead of A_stride; tdls::dynamic_stride (the default) reads A_stride
piv_static_stride	compile-time stride of piv, used instead of piv_stride; tdls::dynamic_stride (the default) reads piv_stride
rhs_static_stride	compile-time stride of y, used instead of rhs_stride; tdls::dynamic_stride (the default) reads rhs_stride

Parameters

[in,out]	A	on entry the matrix (pre-offset by the caller), on exit its factorization
[in]	A_stride	element stride of A (external mode)
[out]	piv	permutation storage (always caller-provided)
[in]	piv_stride	element stride of piv (external mode)
[in,out]	y	right-hand side on entry, solution on exit
[in]	rhs_stride	element stride of y (external mode)
[out]	oot_count	number of columns whose best in-tile pivot fell below oot_pivot_threshold

Returns

false on a singular matrix (y left partially updated).

solve_inplace_multirhs() [1/2]

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int nrhs, bool internal_rhs, bool internal_piv, bool internal_matrix, int pass_width = 0, int A_static_stride = dynamic_stride, int piv_static_stride = dynamic_stride, int rhs_static_stride = dynamic_stride>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool tdls::TiledLUppSolverStatic< T, N, Config >::solve_inplace_multirhs ( T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT y, const int rhs_stride, const std::size_t xcol_stride )

inlinestaticnodiscardconstexprnoexcept

Diagnostics-free solve_inplace_multirhs overload: no out-of-tile out-parameter at all.

Template Parameters

nrhs	total number of right-hand-side columns
internal_rhs	residency of y
internal_piv	residency of piv
internal_matrix	residency of A
pass_width	columns solved together per substitution pass; the default 0, or any value >= nrhs, means one single pass
A_static_stride	compile-time stride of A, used instead of A_stride; tdls::dynamic_stride (the default) reads A_stride
piv_static_stride	compile-time stride of piv, used instead of piv_stride; tdls::dynamic_stride (the default) reads piv_stride
rhs_static_stride	compile-time stride of y, used instead of rhs_stride; tdls::dynamic_stride (the default) reads rhs_stride

Parameters

[in,out]	A	on entry the matrix (pre-offset by the caller), on exit its factorization
[in]	A_stride	element stride of A (external mode)
[out]	piv	permutation storage (always caller-provided)
[in]	piv_stride	element stride of piv (external mode)
[in,out]	y	nrhs right-hand-side columns on entry, nrhs solution columns on exit
[in]	rhs_stride	element stride of y (external mode)
[in]	xcol_stride	element stride between columns of y (external mode)

Returns

false on a singular matrix (y left untouched).

solve_inplace_multirhs() [2/2]

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int nrhs, bool internal_rhs, bool internal_piv, bool internal_matrix, int pass_width = 0, bool oot_diagnostics = true, int A_static_stride = dynamic_stride, int piv_static_stride = dynamic_stride, int rhs_static_stride = dynamic_stride>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool tdls::TiledLUppSolverStatic< T, N, Config >::solve_inplace_multirhs ( T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT y, const int rhs_stride, const std::size_t xcol_stride, int & oot_count )

inlinestaticnodiscardconstexprnoexcept

factorize + substitute_inplace_multirhs in one call: y holds the nrhs unpermuted right-hand-side columns on entry and the nrhs solutions on exit.

Unlike solve_inplace, the forward pass is not folded into the factorization: a pass amortizes the forward tile loads across its columns, which removes most of what the folding saves.

Template Parameters

nrhs	total number of right-hand-side columns
internal_rhs	residency of y
internal_piv	residency of piv
internal_matrix	residency of A
pass_width	columns solved together per substitution pass; the default 0, or any value >= nrhs, means one single pass
oot_diagnostics	compile the out-of-tile counter in or out
A_static_stride	compile-time stride of A, used instead of A_stride; tdls::dynamic_stride (the default) reads A_stride
piv_static_stride	compile-time stride of piv, used instead of piv_stride; tdls::dynamic_stride (the default) reads piv_stride
rhs_static_stride	compile-time stride of y, used instead of rhs_stride; tdls::dynamic_stride (the default) reads rhs_stride

Parameters

[in,out]	A	on entry the matrix (pre-offset by the caller), on exit its factorization
[in]	A_stride	element stride of A (external mode)
[out]	piv	permutation storage (always caller-provided)
[in]	piv_stride	element stride of piv (external mode)
[in,out]	y	nrhs right-hand-side columns on entry, nrhs solution columns on exit
[in]	rhs_stride	element stride of y (external mode)
[in]	xcol_stride	element stride between columns of y (external mode)
[out]	oot_count	number of columns whose best in-tile pivot fell below oot_pivot_threshold

Returns

false on a singular matrix (y left untouched).

solve_multirhs() [1/2]

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int nrhs, bool internal_rhs, bool internal_piv, bool internal_matrix, int pass_width = 0, int A_static_stride = dynamic_stride, int piv_static_stride = dynamic_stride, int rhs_static_stride = dynamic_stride>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool tdls::TiledLUppSolverStatic< T, N, Config >::solve_multirhs ( T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride, const T *TDLS_RESTRICT b, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride )

inlinestaticnodiscardconstexprnoexcept

Diagnostics-free solve_multirhs overload: no out-of-tile out-parameter at all.

Template Parameters

nrhs	total number of right-hand-side columns
internal_rhs	residency of b and x
internal_piv	residency of piv
internal_matrix	residency of A
pass_width	columns solved together per substitution pass; the default 0, or any value >= nrhs, means one single pass
A_static_stride	compile-time stride of A, used instead of A_stride; tdls::dynamic_stride (the default) reads A_stride
piv_static_stride	compile-time stride of piv, used instead of piv_stride; tdls::dynamic_stride (the default) reads piv_stride
rhs_static_stride	compile-time stride of b and x, used instead of rhs_stride; tdls::dynamic_stride (the default) reads rhs_stride

Parameters

[in,out]	A	on entry the matrix (pre-offset by the caller), on exit its factorization
[in]	A_stride	element stride of A (external mode)
[out]	piv	permutation storage (always caller-provided)
[in]	piv_stride	element stride of piv (external mode)
[in]	b	nrhs right-hand-side columns, in original order
[out]	x	nrhs solution columns
[in]	rhs_stride	element stride of b and x (external mode)
[in]	xcol_stride	element stride between columns of b and x (external mode)

Returns

false on a singular matrix.

solve_multirhs() [2/2]

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int nrhs, bool internal_rhs, bool internal_piv, bool internal_matrix, int pass_width = 0, bool oot_diagnostics = true, int A_static_stride = dynamic_stride, int piv_static_stride = dynamic_stride, int rhs_static_stride = dynamic_stride>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool tdls::TiledLUppSolverStatic< T, N, Config >::solve_multirhs ( T *TDLS_RESTRICT A, const int A_stride, int *TDLS_RESTRICT piv, const int piv_stride, const T *TDLS_RESTRICT b, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride, int & oot_count )

inlinestaticnodiscardconstexprnoexcept

factorize + substitute_multirhs in one call.

Template Parameters

nrhs	total number of right-hand-side columns
internal_rhs	residency of b and x
internal_piv	residency of piv
internal_matrix	residency of A
pass_width	columns solved together per substitution pass; the default 0, or any value >= nrhs, means one single pass
oot_diagnostics	compile the out-of-tile counter in or out
A_static_stride	compile-time stride of A, used instead of A_stride; tdls::dynamic_stride (the default) reads A_stride
piv_static_stride	compile-time stride of piv, used instead of piv_stride; tdls::dynamic_stride (the default) reads piv_stride
rhs_static_stride	compile-time stride of b and x, used instead of rhs_stride; tdls::dynamic_stride (the default) reads rhs_stride

Parameters

[in,out]	A	on entry the matrix (pre-offset by the caller), on exit its factorization (usable for further substitute* calls)
[in]	A_stride	element stride of A (external mode)
[out]	piv	permutation storage (always caller-provided)
[in]	piv_stride	element stride of piv (external mode)
[in]	b	nrhs right-hand-side columns, in original order
[out]	x	nrhs solution columns
[in]	rhs_stride	element stride of b and x (external mode)
[in]	xcol_stride	element stride between columns of b and x (external mode)
[out]	oot_count	number of columns whose best in-tile pivot fell below oot_pivot_threshold

Returns

false on a singular matrix.

store_tile()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int row_extent, int col_extent, bool internal_matrix>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::store_tile ( T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT prow, const int col0, const T *TDLS_RESTRICT t )

inlinestaticconstexprnoexcept

Store an RxC tile through a cached physical-row segment.

Template Parameters

row_extent	tile row extent
col_extent	tile column extent
internal_matrix	residency of A

Parameters

[in,out]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A (external mode)
[in]	prow	physical rows of the tile
[in]	col0	first global column of the tile
[in]	t	source register tile

store_tile_piv()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int row_extent, int col_extent, bool internal_piv, bool internal_matrix>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::store_tile_piv ( T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, const int row0, const int col0, const T *TDLS_RESTRICT t )

inlinestaticconstexprnoexcept

Store an RxC tile, reading the permutation inline (one read per row).

Template Parameters

row_extent	tile row extent
col_extent	tile column extent
internal_piv	residency of piv
internal_matrix	residency of A

Parameters

[in,out]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A (external mode)
[in]	piv	permutation (logical -> physical row)
[in]	piv_stride	element stride of piv (external mode)
[in]	row0	first global (logical) row of the tile
[in]	col0	first global column of the tile
[in]	t	source register tile

substitute()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<bool internal_rhs, bool internal_piv, bool internal_matrix, int A_static_stride = dynamic_stride, int piv_static_stride = dynamic_stride, int rhs_static_stride = dynamic_stride>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::substitute ( const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, const T *TDLS_RESTRICT b, T *TDLS_RESTRICT x, const int rhs_stride )

inlinestaticconstexprnoexcept

Solve x := U^-1 L^-1 P b from a prior factorize (column-major layout: x := P^T L^-T U^-T b). b and x must not alias (use substitute_inplace for the aliased single-buffer pattern).

Template Parameters

internal_rhs	true: b and x are caller-local T[N], strides ignored; false: remote, indexed with rhs_stride
internal_piv	residency of piv
internal_matrix	residency of A
A_static_stride	compile-time stride of A, used instead of A_stride; tdls::dynamic_stride (the default) reads A_stride
piv_static_stride	compile-time stride of piv, used instead of piv_stride; tdls::dynamic_stride (the default) reads piv_stride
rhs_static_stride	compile-time stride of b and x, used instead of rhs_stride; tdls::dynamic_stride (the default) reads rhs_stride

Parameters

[in]	A	factored matrix produced by factorize
[in]	A_stride	element stride of A (external mode)
[in]	piv	permutation produced by factorize
[in]	piv_stride	element stride of piv (external mode)
[in]	b	right-hand side, in original (unpermuted) order
[out]	x	solution
[in]	rhs_stride	element stride of b and x (external mode)

substitute_canonical()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<bool internal_rhs, bool internal_piv, bool internal_matrix, int A_static_stride = dynamic_stride, int piv_static_stride = dynamic_stride, int rhs_static_stride = dynamic_stride>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::substitute_canonical ( const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, const int col, T *TDLS_RESTRICT x, const int rhs_stride )

inlinestaticconstexprnoexcept

Solve with b = e_col generated on the fly: the consistent-tangent-operator path.

Thin alias of the canonical pass engine below with pass_width = 1, which collapses to exactly the single-column code, so there is no separate implementation to maintain.

Template Parameters

internal_rhs	residency of x
internal_piv	residency of piv
internal_matrix	residency of A
A_static_stride	compile-time stride of A, used instead of A_stride; tdls::dynamic_stride (the default) reads A_stride
piv_static_stride	compile-time stride of piv, used instead of piv_stride; tdls::dynamic_stride (the default) reads piv_stride
rhs_static_stride	compile-time stride of x, used instead of rhs_stride; tdls::dynamic_stride (the default) reads rhs_stride

Parameters

[in]	A	factored matrix produced by factorize
[in]	A_stride	element stride of A (external mode)
[in]	piv	permutation produced by factorize
[in]	piv_stride	element stride of piv (external mode)
[in]	col	index of the canonical column e_col
[out]	x	solution
[in]	rhs_stride	element stride of x (external mode)

substitute_canonical_multirhs()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int nrhs, bool internal_rhs, bool internal_piv, bool internal_matrix, int pass_width = 0, int A_static_stride = dynamic_stride, int piv_static_stride = dynamic_stride, int rhs_static_stride = dynamic_stride>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::substitute_canonical_multirhs ( const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, const int col0, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride )

inlinestaticconstexprnoexcept

nrhs canonical columns e_col0 .. e_{col0+nrhs-1} solved from a prior factorize: the columns of the inverse that a consistent tangent operator needs.

Column w lives at x + w*N (internal) or x + w*xcol_stride (remote). The columns are solved in passes of pass_width columns; within a pass, every L/U tile is loaded once for all the columns of the pass. Per-column arithmetic is identical whatever the cutting, and identical to nrhs separate substitute_canonical calls: results match them bitwise when the compiler contracts multiply-adds alike in both (see the class documentation; nrhs = 1 collapses to substitute_canonical exactly).

Template Parameters

nrhs	number of consecutive canonical columns
internal_rhs	residency of x
internal_piv	residency of piv
internal_matrix	residency of A
pass_width	columns solved together per pass; the default 0, or any value >= nrhs, means one single pass of nrhs columns
A_static_stride	compile-time stride of A, used instead of A_stride; tdls::dynamic_stride (the default) reads A_stride
piv_static_stride	compile-time stride of piv, used instead of piv_stride; tdls::dynamic_stride (the default) reads piv_stride
rhs_static_stride	compile-time stride of x, used instead of rhs_stride; tdls::dynamic_stride (the default) reads rhs_stride

Parameters

[in]	A	factored matrix produced by factorize
[in]	A_stride	element stride of A (external mode)
[in]	piv	permutation produced by factorize
[in]	piv_stride	element stride of piv (external mode)
[in]	col0	index of the first canonical column
[out]	x	nrhs solution columns
[in]	rhs_stride	element stride of x (external mode)
[in]	xcol_stride	element stride between columns of x (external mode)

substitute_canonical_multirhs_pass()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int pass_width, bool internal_rhs, bool internal_piv, bool internal_matrix>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::substitute_canonical_multirhs_pass ( const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, const int col0, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride )

inlinestaticconstexprnoexcept

One pass of the canonical multi right-hand-side substitution: pass_width canonical columns e_col0 .. e_{col0+pass_width-1} generated in permuted order into x, then one triangular sweep for all pass_width columns together. Under the column-major layout the columns are generated unpermuted, swept through U^T then L^T, then permuted (x = P^T w). Internal engine of substitute_canonical_multirhs.

Template Parameters

pass_width	number of columns of this pass
internal_rhs	residency of x
internal_piv	residency of piv
internal_matrix	residency of A

Parameters

[in]	A	factored matrix produced by factorize
[in]	A_stride	element stride of A (external mode)
[in]	piv	permutation produced by factorize
[in]	piv_stride	element stride of piv (external mode)
[in]	col0	index of the first canonical column
[out]	x	pass_width solution columns
[in]	rhs_stride	element stride of x (external mode)
[in]	xcol_stride	element stride between columns of x (external mode)

substitute_inplace()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<bool internal_rhs, bool internal_piv, bool internal_matrix, int A_static_stride = dynamic_stride, int piv_static_stride = dynamic_stride, int rhs_static_stride = dynamic_stride>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::substitute_inplace ( const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride )

inlinestaticconstexprnoexcept

Solve in place: x already holds the unpermuted RHS on entry and the solution on exit.

The permutation is applied in place by cycle decomposition, each cycle rotated once from its smallest index. Up to N = 64 a bitmask tracks the visited rows (a single 32- or 64-bit register); beyond, a cycle-leader scan is used instead (zero storage, an extra integer walk of each orbit). Both paths move the same values in the same order. Under the column-major layout, the sweeps run first on the unpermuted x and the inverse rotation (x = P^T w) comes last.

Template Parameters

internal_rhs	residency of x
internal_piv	residency of piv
internal_matrix	residency of A
A_static_stride	compile-time stride of A, used instead of A_stride; tdls::dynamic_stride (the default) reads A_stride
piv_static_stride	compile-time stride of piv, used instead of piv_stride; tdls::dynamic_stride (the default) reads piv_stride
rhs_static_stride	compile-time stride of x, used instead of rhs_stride; tdls::dynamic_stride (the default) reads rhs_stride

Parameters

[in]	A	factored matrix produced by factorize
[in]	A_stride	element stride of A (external mode)
[in]	piv	permutation produced by factorize
[in]	piv_stride	element stride of piv (external mode)
[in,out]	x	right-hand side on entry, solution on exit
[in]	rhs_stride	element stride of x (external mode)

substitute_inplace_multirhs()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int nrhs, bool internal_rhs, bool internal_piv, bool internal_matrix, int pass_width = 0, int A_static_stride = dynamic_stride, int piv_static_stride = dynamic_stride, int rhs_static_stride = dynamic_stride>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::substitute_inplace_multirhs ( const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride )

inlinestaticconstexprnoexcept

nrhs columns solved in place: x holds the nrhs unpermuted right-hand-side columns on entry and the nrhs solutions on exit.

The permutation is applied by the same cycle decomposition as substitute_inplace. The columns are solved in passes of pass_width columns; within a pass, every L/U tile is loaded once for all the columns of the pass. Per-column values are identical whatever the cutting, and match nrhs separate substitute_inplace calls bitwise when the compiler contracts multiply-adds alike in both (see the class documentation; nrhs = 1 collapses to substitute_inplace exactly).

Template Parameters

nrhs	total number of right-hand-side columns
internal_rhs	residency of x
internal_piv	residency of piv
internal_matrix	residency of A
pass_width	columns solved together per pass; the default 0, or any value >= nrhs, means one single pass of nrhs columns
A_static_stride	compile-time stride of A, used instead of A_stride; tdls::dynamic_stride (the default) reads A_stride
piv_static_stride	compile-time stride of piv, used instead of piv_stride; tdls::dynamic_stride (the default) reads piv_stride
rhs_static_stride	compile-time stride of x, used instead of rhs_stride; tdls::dynamic_stride (the default) reads rhs_stride

Parameters

[in]	A	factored matrix produced by factorize
[in]	A_stride	element stride of A (external mode)
[in]	piv	permutation produced by factorize
[in]	piv_stride	element stride of piv (external mode)
[in,out]	x	nrhs right-hand-side columns on entry, nrhs solution columns on exit
[in]	rhs_stride	element stride of x (external mode)
[in]	xcol_stride	element stride between columns of x (external mode)

substitute_inplace_multirhs_pass()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int pass_width, bool internal_rhs, bool internal_piv, bool internal_matrix>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::substitute_inplace_multirhs_pass ( const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride )

inlinestaticconstexprnoexcept

One pass of the in-place multi right-hand-side substitution: pass_width columns of x permuted by the cycle decomposition of substitute_inplace (each move carrying the pass_width column entries of its row), then one triangular sweep for all pass_width columns together. Under the column-major layout the sweeps come first, then the cycle decomposition applies x = P^T w. Internal engine of substitute_inplace_multirhs.

Template Parameters

pass_width	number of columns of this pass
internal_rhs	residency of x
internal_piv	residency of piv
internal_matrix	residency of A

Parameters

[in]	A	factored matrix produced by factorize
[in]	A_stride	element stride of A (external mode)
[in]	piv	permutation produced by factorize
[in]	piv_stride	element stride of piv (external mode)
[in,out]	x	pass_width right-hand-side columns on entry, pass_width solution columns on exit
[in]	rhs_stride	element stride of x (external mode)
[in]	xcol_stride	element stride between columns of x (external mode)

substitute_multirhs()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int nrhs, bool internal_rhs, bool internal_piv, bool internal_matrix, int pass_width = 0, int A_static_stride = dynamic_stride, int piv_static_stride = dynamic_stride, int rhs_static_stride = dynamic_stride>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::substitute_multirhs ( const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, const T *TDLS_RESTRICT b, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride )

inlinestaticconstexprnoexcept

Solve nrhs right-hand-side columns from a prior factorize: X := U^-1 L^-1 P B (column-major layout: X := P^T L^-T U^-T B), column by column. b and x must not alias (use substitute_inplace_multirhs for the aliased single-buffer pattern).

Column w lives at b/x + w*N (internal) or b/x + w*xcol_stride (remote); b and x share both strides, as they share rhs_stride in substitute. The columns are solved in passes of pass_width columns; within a pass, every L/U tile is loaded once for all the columns of the pass. Per-column arithmetic is identical whatever the cutting, and identical to nrhs separate substitute calls: results match them bitwise when the compiler contracts multiply-adds alike in both (see the class documentation; nrhs = 1 collapses to substitute exactly).

Template Parameters

nrhs	total number of right-hand-side columns
internal_rhs	residency of b and x
internal_piv	residency of piv
internal_matrix	residency of A
pass_width	columns solved together per pass; the default 0, or any value >= nrhs, means one single pass of nrhs columns
A_static_stride	compile-time stride of A, used instead of A_stride; tdls::dynamic_stride (the default) reads A_stride
piv_static_stride	compile-time stride of piv, used instead of piv_stride; tdls::dynamic_stride (the default) reads piv_stride
rhs_static_stride	compile-time stride of b and x, used instead of rhs_stride; tdls::dynamic_stride (the default) reads rhs_stride

Parameters

[in]	A	factored matrix produced by factorize
[in]	A_stride	element stride of A (external mode)
[in]	piv	permutation produced by factorize
[in]	piv_stride	element stride of piv (external mode)
[in]	b	nrhs right-hand-side columns, in original (unpermuted) order
[out]	x	nrhs solution columns
[in]	rhs_stride	element stride of b and x (external mode)
[in]	xcol_stride	element stride between columns of b and x (external mode)

substitute_multirhs_pass()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int pass_width, bool internal_rhs, bool internal_piv, bool internal_matrix>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::substitute_multirhs_pass ( const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, const T *TDLS_RESTRICT b, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride )

inlinestaticconstexprnoexcept

One pass of the multi right-hand-side substitution: pass_width columns of b gathered in permuted order into x, then one triangular sweep for all pass_width columns together. Under the column-major layout the columns are copied unpermuted, swept through U^T then L^T, then permuted (x = P^T w). Internal engine of substitute_multirhs.

Template Parameters

pass_width	number of columns of this pass
internal_rhs	residency of b and x
internal_piv	residency of piv
internal_matrix	residency of A

Parameters

[in]	A	factored matrix produced by factorize
[in]	A_stride	element stride of A (external mode)
[in]	piv	permutation produced by factorize
[in]	piv_stride	element stride of piv (external mode)
[in]	b	pass_width right-hand-side columns, in original (unpermuted) order
[out]	x	pass_width solution columns
[in]	rhs_stride	element stride of b and x (external mode)
[in]	xcol_stride	element stride between columns of b and x (external mode)

transposed_bwd_only()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int pass_width, bool internal_rhs, bool internal_piv, bool internal_matrix>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::transposed_bwd_only ( const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride )

inlinestaticconstexprnoexcept

Transposed backward pass alone, used by transposed_fwd_bwd and by solve_inplace (whose forward pass happens inside the factorization).

Template Parameters

pass_width	number of columns processed together
internal_rhs	residency of x
internal_piv	residency of piv
internal_matrix	residency of A

Parameters

[in]	A	factored matrix (caller-pre-offset)
[in]	A_stride	element stride of A (external mode)
[in]	piv	permutation from factorize
[in]	piv_stride	element stride of piv (external mode)
[in,out]	x	solution column(s)
[in]	rhs_stride	element stride of x (external mode)
[in]	xcol_stride	element stride between columns of x

transposed_bwd_step()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int k_extent, int pass_width, bool internal_rhs, bool internal_piv, bool internal_matrix>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::transposed_bwd_step ( const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride, const int k )

inlinestaticconstexprnoexcept

Transposed backward step: pull the trailing contributions with L^T, then L^T-solve the diagonal tile's segment.

Template Parameters

k_extent	extent of the diagonal tile of the step
pass_width	number of columns processed together
internal_rhs	residency of x
internal_piv	residency of piv
internal_matrix	residency of A

Parameters

[in]	A	factored matrix (caller-pre-offset)
[in]	A_stride	element stride of A (external mode)
[in]	piv	permutation from factorize
[in]	piv_stride	element stride of piv (external mode)
[in,out]	x	solution column(s)
[in]	rhs_stride	element stride of x (external mode)
[in]	xcol_stride	element stride between columns of x
[in]	k	step index (k0 = k*tile_size)

transposed_fwd_bwd()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int pass_width, bool internal_rhs, bool internal_piv, bool internal_matrix>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::transposed_fwd_bwd ( const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride )

inlinestaticconstexprnoexcept

Transposed triangular sweeps on unpermuted column(s) x: U^T z = x, then L^T w = z, w left in x (x = P^T w is the caller's last step).

Template Parameters

pass_width	number of columns processed together
internal_rhs	residency of x
internal_piv	residency of piv
internal_matrix	residency of A

Parameters

[in]	A	factored matrix (caller-pre-offset)
[in]	A_stride	element stride of A (external mode)
[in]	piv	permutation from factorize
[in]	piv_stride	element stride of piv (external mode)
[in,out]	x	right-hand side column(s) on entry, w on exit
[in]	rhs_stride	element stride of x (external mode)
[in]	xcol_stride	element stride between columns of x

transposed_fwd_step()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int k_extent, int pass_width, bool internal_rhs, bool internal_piv, bool internal_matrix>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::transposed_fwd_step ( const T *TDLS_RESTRICT A, const int A_stride, const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride, const int k )

inlinestaticconstexprnoexcept

Transposed forward step: U^T-solve the diagonal tile's segment, then push it into the following segments.

Template Parameters

k_extent	extent of the diagonal tile of the step
pass_width	number of columns processed together
internal_rhs	residency of x
internal_piv	residency of piv
internal_matrix	residency of A

Parameters

[in]	A	factored matrix (caller-pre-offset)
[in]	A_stride	element stride of A (external mode)
[in]	piv	permutation from factorize
[in]	piv_stride	element stride of piv (external mode)
[in,out]	x	right-hand side column(s)
[in]	rhs_stride	element stride of x (external mode)
[in]	xcol_stride	element stride between columns of x
[in]	k	step index (k0 = k*tile_size)

transposed_pull_one()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int k_extent, int m_extent, int pass_width, bool internal_rhs>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::transposed_pull_one ( const T *TDLS_RESTRICT l, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride, const int k0, const int m0 )

inlinestaticconstexprnoexcept

Backward pull with L^T: x_k segment -= L(m,k)^T * x_m.

Template Parameters

k_extent	extent of the updated segment's tile
m_extent	extent of the trailing segment's tile
pass_width	number of columns processed together
internal_rhs	residency of x

Parameters

[in]	l	L(m,k) tile (m_extent rows, k_extent columns)
[in,out]	x	solution column(s)
[in]	rhs_stride	element stride of x (external mode)
[in]	xcol_stride	element stride between columns of x
[in]	k0	first global index of the updated segment
[in]	m0	first global index of the trailing segment

transposed_push_one()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int k_extent, int m_extent, int pass_width, bool internal_rhs>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::transposed_push_one ( const T *TDLS_RESTRICT u, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride, const int k0, const int m0 )

inlinestaticconstexprnoexcept

Forward push with U^T: x_m segment -= U(k,m)^T * x_k.

Template Parameters

k_extent	extent of the solved segment's tile
m_extent	extent of the target segment's tile
pass_width	number of columns processed together
internal_rhs	residency of x

Parameters

[in]	u	U(k,m) tile (k_extent rows, m_extent columns)
[in,out]	x	right-hand side column(s)
[in]	rhs_stride	element stride of x (external mode)
[in]	xcol_stride	element stride between columns of x
[in]	k0	first global index of the solved segment
[in]	m0	first global index of the target segment

transposed_solve_tile_lower_unit()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int k_extent, int pass_width, bool internal_rhs>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::transposed_solve_tile_lower_unit ( const T *TDLS_RESTRICT l, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride, const int k0 )

inlinestaticconstexprnoexcept

In-tile backward solve with L^T (unit diagonal: no division and no multiplication by the diagonal).

Template Parameters

k_extent	extent of the diagonal tile
pass_width	number of columns processed together
internal_rhs	residency of x

Parameters

[in]	l	strictly lower part of the diagonal tile
[in,out]	x	solution column(s)
[in]	rhs_stride	element stride of x (external mode)
[in]	xcol_stride	element stride between columns of x
[in]	k0	first global index of the segment

transposed_solve_tile_upper()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int k_extent, int pass_width, bool internal_rhs>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::transposed_solve_tile_upper ( const T *TDLS_RESTRICT t, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride, const int k0 )

inlinestaticconstexprnoexcept

In-tile forward solve with U^T: x_k segment := U(k,k)^-T x_k (the diagonal of the tile holds the pivot reciprocals).

Template Parameters

k_extent	extent of the diagonal tile
pass_width	number of columns processed together
internal_rhs	residency of x

Parameters

[in]	t	factored diagonal tile
[in,out]	x	right-hand side column(s)
[in]	rhs_stride	element stride of x (external mode)
[in]	xcol_stride	element stride between columns of x
[in]	k0	first global index of the segment

unpermute()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int pass_width, bool internal_rhs, bool internal_piv>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::unpermute ( const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride )

inlinestaticconstexprnoexcept

x := P^T x, predicated for an internal x, by cycles for a remote one.

Template Parameters

pass_width	number of columns of x
internal_rhs	residency of x
internal_piv	residency of piv

Parameters

[in]	piv	permutation from factorize
[in]	piv_stride	element stride of piv (external mode)
[in,out]	x	w on entry, the solution on exit
[in]	rhs_stride	element stride of x (external mode)
[in]	xcol_stride	element stride between columns of x

unpermute_cycles()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int pass_width, bool internal_rhs, bool internal_piv>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::unpermute_cycles ( const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const std::size_t xcol_stride )

inlinestaticconstexprnoexcept

x := P^T x in place (x[piv[i]] receives x[i]), the last step of the transposed solve, by cycle decomposition: the inverse of the rotation of substitute_inplace, each cycle carried once from its smallest index. Up to N = 64 a bitmask tracks the visited rows; beyond, a cycle-leader scan. Values are moved, never computed.

Template Parameters

pass_width	number of columns of x
internal_rhs	residency of x
internal_piv	residency of piv

Parameters

[in]	piv	permutation from factorize
[in]	piv_stride	element stride of piv (external mode)
[in,out]	x	w on entry, the solution on exit
[in]	rhs_stride	element stride of x (external mode)
[in]	xcol_stride	element stride between columns of x

unpermute_predicated()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int pass_width, bool internal_piv>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::unpermute_predicated ( const int *TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x )

inlinestaticconstexprnoexcept

x := P^T x for an internal x, predicated over the unrolled index range: x must never be dynamically indexed or it is demoted to local memory. Written as the gather of substitute, a scalar selected over the sources (x[j] receives w[i] where piv[i] == j), which compiles to selects; a predicated store into x would compile to branches, divergent across the threads of a warp.

Template Parameters

pass_width	number of columns of x
internal_piv	residency of piv

Parameters

[in]	piv	permutation from factorize
[in]	piv_stride	element stride of piv (external mode)
[in,out]	x	w on entry, the solution on exit (internal)

Member Data Documentation

full_tiles

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

int tdls::TiledLUppSolverStatic< T, N, Config >::full_tiles = N / tile_size

staticconstexpr

full tiles per dimension

schedule

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

Schedule tdls::TiledLUppSolverStatic< T, N, Config >::schedule

staticconstexpr

Initial value:

=
        Config.schedule

elimination schedule (RightLooking or LeftLooking)

---

The documentation for this struct was generated from the following file:

• include/tdls/solvers/tiled_lupp/solver_static.hpp