Tiled dense LU factorization with logical partial pivoting and out-of-tile pivot recovery, solving one NxN system per call. More...

#include <solver_static.hpp>

Public Types
using	Operations = TiledLUppTileOperations<T, tile_size, Config.unroll_inner>
	Tile micro-kernels instantiated for this configuration.

Static Public Member Functions
template<int row_extent, int col_extent, bool internal_matrix>
TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void	load_tile (const T TDLS_RESTRICT A, const int A_stride, const int TDLS_RESTRICT prow, const int col0, T *TDLS_RESTRICT t) noexcept
	Load an RxC tile through a cached physical-row segment.

template<int row_extent, int col_extent, bool internal_matrix>
TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void	store_tile (T TDLS_RESTRICT A, const int A_stride, const int TDLS_RESTRICT prow, const int col0, const T *TDLS_RESTRICT t) noexcept
	Store an RxC tile through a cached physical-row segment.

template<int row_extent, int col_extent, bool internal_piv, bool internal_matrix>
TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void	load_tile_piv (const T TDLS_RESTRICT A, const int A_stride, const int TDLS_RESTRICT piv, const int piv_stride, const int row0, const int col0, T *TDLS_RESTRICT t) noexcept
	Load an RxC tile, reading the permutation inline (one read per row).

template<int row_extent, int col_extent, bool internal_piv, bool internal_matrix>
TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void	store_tile_piv (T TDLS_RESTRICT A, const int A_stride, const int TDLS_RESTRICT piv, const int piv_stride, const int row0, const int col0, const T *TDLS_RESTRICT t) noexcept
	Store an RxC tile, reading the permutation inline (one read per row).

template<int row_extent, bool internal_piv, bool internal_matrix>
TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void	load_tile_piv_lower (const T TDLS_RESTRICT A, const int A_stride, const int TDLS_RESTRICT piv, const int piv_stride, const int row0, const int col0, T *TDLS_RESTRICT t) noexcept
	Triangular variant of the diagonal-tile load for the forward substitution: only the strict lower triangle of L\U is read.

template<int row_extent, bool internal_piv, bool internal_matrix>
TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void	load_tile_piv_upper (const T TDLS_RESTRICT A, const int A_stride, const int TDLS_RESTRICT piv, const int piv_stride, const int row0, const int col0, T *TDLS_RESTRICT t) noexcept
	Triangular variant of the diagonal-tile load for the backward substitution: only the upper triangle including the diagonal is read.

template<int k_extent, bool internal_piv, bool internal_matrix, bool oot_diagnostics, bool fuse_rhs, bool internal_rhs>
TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr bool	factor_diag_column (T TDLS_RESTRICT A, const int A_stride, int TDLS_RESTRICT piv, const int piv_stride, const int k0, T TDLS_RESTRICT tile, int &oot_count, const int c, T TDLS_RESTRICT y, const int rhs_stride) noexcept
	One column step of the diagonal-tile factorization: pivot search (in-tile, then out-of-tile recovery), permutation update, row swap or cross-tile pull, column elimination.

template<int k_extent, bool internal_piv, bool internal_matrix, bool oot_diagnostics, bool fuse_rhs = false, bool internal_rhs = true>
TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr bool	factor_diag_tile (T TDLS_RESTRICT A, const int A_stride, int TDLS_RESTRICT piv, const int piv_stride, const int k0, T TDLS_RESTRICT tile, int &oot_count, T TDLS_RESTRICT y=nullptr, const int rhs_stride=1) noexcept
	Factor the KExKE diagonal tile in registers, with out-of-tile pivot recovery (drives the per-column loop of factor_diag_column).

template<int k_extent, int j_extent, bool internal_matrix>
TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void	rl_trsm_right_one (T TDLS_RESTRICT A, const int A_stride, const int TDLS_RESTRICT pk, const T *TDLS_RESTRICT tile, const int j0) noexcept
	RL: one row-panel tile update, Akj := L^-1 Akj.

template<int k_extent, int i_extent, int j_extent, bool internal_matrix>
TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void	rl_schur_one (T TDLS_RESTRICT A, const int A_stride, const int TDLS_RESTRICT pk, const int TDLS_RESTRICT pi, const T TDLS_RESTRICT Aik, const int j0) noexcept
	RL: one Schur-complement tile update, Aij -= Aik * Akj, streaming the factored Akj row by row from remote memory (it is not worth a third register tile).

template<int k_extent, int i_extent, bool internal_piv, bool internal_matrix, bool fuse_rhs = false, bool internal_rhs = true>
TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void	rl_update_row_one (T TDLS_RESTRICT A, const int A_stride, const int TDLS_RESTRICT piv, const int piv_stride, const int TDLS_RESTRICT pk, const T TDLS_RESTRICT tile, const int k, const int i0, T *TDLS_RESTRICT y=nullptr, const int rhs_stride=1) noexcept
	RL: TRSM down + Schur sweep of one row block below the diagonal, with the optional fused forward-substitution push.

template<int k_extent, bool internal_piv, bool internal_matrix, bool oot_diagnostics, bool fuse_rhs = false, bool internal_rhs = true>
TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr bool	rl_step (T TDLS_RESTRICT A, const int A_stride, int TDLS_RESTRICT piv, const int piv_stride, const int k, int &oot_count, T *TDLS_RESTRICT y=nullptr, const int rhs_stride=1) noexcept
	RL: one full factorization step (diagonal tile + trailing updates).

template<int row_extent, int col_extent, bool internal_piv, bool internal_matrix>
TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void	ll_correct_tile (const T TDLS_RESTRICT A, const int A_stride, const int TDLS_RESTRICT piv, const int piv_stride, const int row0, const int col0, const int k0, T *TDLS_RESTRICT t) noexcept
	LL: t (RExCE, rows row0.., cols col0..) -= sum over prior tiles bj < k0/tile_size of L(row0.., bj) * U(bj, col0..). All prior tiles are full.

template<int k_extent, int i_extent, bool internal_piv, bool internal_matrix, bool fuse_rhs = false, bool internal_rhs = true>
TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void	ll_update_below_one (T TDLS_RESTRICT A, const int A_stride, const int TDLS_RESTRICT piv, const int piv_stride, const T TDLS_RESTRICT tile, const int k0, const int i0, T TDLS_RESTRICT y=nullptr, const int rhs_stride=1) noexcept
	LL: correct + TRSM one L-panel tile below the diagonal, with the optional fused forward-substitution push.

template<int k_extent, int j_extent, bool internal_piv, bool internal_matrix>
TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void	ll_update_right_one (T TDLS_RESTRICT A, const int A_stride, const int TDLS_RESTRICT piv, const int piv_stride, const T *TDLS_RESTRICT tile, const int k0, const int j0) noexcept
	LL: correct + TRSM one U-panel tile right of the diagonal.

template<int k_extent, bool internal_piv, bool internal_matrix, bool oot_diagnostics, bool fuse_rhs = false, bool internal_rhs = true>
TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr bool	ll_step (T TDLS_RESTRICT A, const int A_stride, int TDLS_RESTRICT piv, const int piv_stride, const int k, int &oot_count, T *TDLS_RESTRICT y=nullptr, const int rhs_stride=1) noexcept
	LL: one full factorization step (correct + factor the diagonal tile, then its L and U panels).

template<bool internal_piv, bool internal_matrix, bool oot_diagnostics = true, bool fuse_rhs = false, bool internal_rhs = true>
TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr bool	factorize (T TDLS_RESTRICT A, const int A_stride, int TDLS_RESTRICT piv, const int piv_stride, int &oot_count, T *TDLS_RESTRICT y=nullptr, const int rhs_stride=1) noexcept
	Factor A := PLU in place.

template<bool internal_piv, bool internal_matrix>
TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr bool	factorize (T TDLS_RESTRICT A, const int A_stride, int TDLS_RESTRICT piv, const int piv_stride) noexcept
	Diagnostics-free factorize overload: no out-of-tile out-parameter at all.

template<int k_extent, int m_extent, int pass_width, bool internal_rhs, bool internal_piv, bool internal_matrix>
TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void	fwd_push_one (const T TDLS_RESTRICT A, const int A_stride, const int TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const int xcol_stride, const int k0, const int m0) noexcept
	Forward push: subtract L(m,k) * x_k from the x_m segment, for pass_width columns at once.

template<int k_extent, int pass_width, bool internal_rhs, bool internal_piv, bool internal_matrix>
TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void	fwd_step (const T TDLS_RESTRICT A, const int A_stride, const int TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const int xcol_stride, const int k) noexcept
	Forward step: unit-lower solve the diagonal tile's segment, then push it into the tiles below.

template<int k_extent, int m_extent, int pass_width, bool internal_rhs, bool internal_piv, bool internal_matrix>
TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void	bwd_pull_one (const T TDLS_RESTRICT A, const int A_stride, const int TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const int xcol_stride, const int k0, const int m0) noexcept
	Backward pull: subtract U(k,m) * x_m from the x_k segment, for pass_width columns at once.

template<int k_extent, int pass_width, bool internal_rhs, bool internal_piv, bool internal_matrix>
TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void	bwd_step (const T TDLS_RESTRICT A, const int A_stride, const int TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const int xcol_stride, const int k) noexcept
	Backward step: pull the trailing contributions, then upper-solve the diagonal tile's segment.

template<int pass_width, bool internal_rhs, bool internal_piv, bool internal_matrix>
TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void	bwd_only (const T TDLS_RESTRICT A, const int A_stride, const int TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const int xcol_stride) noexcept
	Backward pass alone, used by fwd_bwd and by solve_inplace (whose forward pass happens inside the factorization).

template<int pass_width, bool internal_rhs, bool internal_piv, bool internal_matrix>
TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void	fwd_bwd (const T TDLS_RESTRICT A, const int A_stride, const int TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const int xcol_stride) noexcept
	Triangular solves on already-permuted column(s) x. pass_width columns are processed per tile visit, so every L/U tile is loaded once for the whole block instead of once per column (pass_width = 1 is the single-RHS case).

template<bool internal_rhs, bool internal_piv, bool internal_matrix>
TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void	substitute (const T TDLS_RESTRICT A, const int A_stride, const int TDLS_RESTRICT piv, const int piv_stride, const T TDLS_RESTRICT b, T TDLS_RESTRICT x, const int rhs_stride) noexcept
	Solve x := U^-1 L^-1 P b from a prior factorize. b and x must not alias (use substitute_inplace for the aliased single-buffer pattern).

template<bool internal_rhs, bool internal_piv, bool internal_matrix>
TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void	substitute_canonical (const T TDLS_RESTRICT A, const int A_stride, const int TDLS_RESTRICT piv, const int piv_stride, const int col, T *TDLS_RESTRICT x, const int rhs_stride) noexcept
	Solve with b = e_col generated on the fly: the consistent-tangent-operator path.

template<int pass_width, bool internal_rhs, bool internal_piv, bool internal_matrix>
TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void	substitute_canonical_multirhs_pass (const T TDLS_RESTRICT A, const int A_stride, const int TDLS_RESTRICT piv, const int piv_stride, const int col0, T *TDLS_RESTRICT x, const int rhs_stride, const int xcol_stride) noexcept
	One pass of the canonical multi right-hand-side substitution: pass_width canonical columns e_col0 .. e_{col0+pass_width-1} generated in permuted order into x, then one triangular sweep for all pass_width columns together. Internal engine of substitute_canonical_multirhs.

template<int nrhs, bool internal_rhs, bool internal_piv, bool internal_matrix, int pass_width = 0>
TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void	substitute_canonical_multirhs (const T TDLS_RESTRICT A, const int A_stride, const int TDLS_RESTRICT piv, const int piv_stride, const int col0, T *TDLS_RESTRICT x, const int rhs_stride, const int xcol_stride) noexcept
	nrhs canonical columns e_col0 .. e_{col0+nrhs-1} solved from a prior factorize: the columns of the inverse that a consistent tangent operator needs.

template<int pass_width, bool internal_rhs, bool internal_piv, bool internal_matrix>
TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void	substitute_multirhs_pass (const T TDLS_RESTRICT A, const int A_stride, const int TDLS_RESTRICT piv, const int piv_stride, const T TDLS_RESTRICT b, T TDLS_RESTRICT x, const int rhs_stride, const int xcol_stride) noexcept
	One pass of the multi right-hand-side substitution: pass_width columns of b gathered in permuted order into x, then one triangular sweep for all pass_width columns together. Internal engine of substitute_multirhs.

template<int nrhs, bool internal_rhs, bool internal_piv, bool internal_matrix, int pass_width = 0>
TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void	substitute_multirhs (const T TDLS_RESTRICT A, const int A_stride, const int TDLS_RESTRICT piv, const int piv_stride, const T TDLS_RESTRICT b, T TDLS_RESTRICT x, const int rhs_stride, const int xcol_stride) noexcept
	Solve nrhs right-hand-side columns from a prior factorize: X := U^-1 L^-1 P B, column by column. b and x must not alias (use substitute_inplace_multirhs for the aliased single-buffer pattern).

template<bool internal_rhs, bool internal_piv, bool internal_matrix>
TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void	substitute_inplace (const T TDLS_RESTRICT A, const int A_stride, const int TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride) noexcept
	Solve in place: x already holds the unpermuted RHS on entry and the solution on exit.

template<int pass_width, bool internal_rhs, bool internal_piv, bool internal_matrix>
TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void	substitute_inplace_multirhs_pass (const T TDLS_RESTRICT A, const int A_stride, const int TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const int xcol_stride) noexcept
	One pass of the in-place multi right-hand-side substitution: pass_width columns of x permuted by the cycle decomposition of substitute_inplace (each move carrying the pass_width column entries of its row), then one triangular sweep for all pass_width columns together. Internal engine of substitute_inplace_multirhs.

template<int nrhs, bool internal_rhs, bool internal_piv, bool internal_matrix, int pass_width = 0>
TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void	substitute_inplace_multirhs (const T TDLS_RESTRICT A, const int A_stride, const int TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const int xcol_stride) noexcept
	nrhs columns solved in place: x holds the nrhs unpermuted right-hand-side columns on entry and the nrhs solutions on exit.

template<bool internal_rhs, bool internal_piv, bool internal_matrix, bool oot_diagnostics = true>
TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr bool	solve (T TDLS_RESTRICT A, const int A_stride, int TDLS_RESTRICT piv, const int piv_stride, const T TDLS_RESTRICT b, T TDLS_RESTRICT x, const int rhs_stride, int &oot_count) noexcept
	factorize + substitute in one call.

template<bool internal_rhs, bool internal_piv, bool internal_matrix>
TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr bool	solve (T TDLS_RESTRICT A, const int A_stride, int TDLS_RESTRICT piv, const int piv_stride, const T TDLS_RESTRICT b, T TDLS_RESTRICT x, const int rhs_stride) noexcept
	Diagnostics-free solve overload: no out-of-tile out-parameter at all.

template<int nrhs, bool internal_rhs, bool internal_piv, bool internal_matrix, int pass_width = 0, bool oot_diagnostics = true>
TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr bool	solve_multirhs (T TDLS_RESTRICT A, const int A_stride, int TDLS_RESTRICT piv, const int piv_stride, const T TDLS_RESTRICT b, T TDLS_RESTRICT x, const int rhs_stride, const int xcol_stride, int &oot_count) noexcept
	factorize + substitute_multirhs in one call.

template<int nrhs, bool internal_rhs, bool internal_piv, bool internal_matrix, int pass_width = 0>
TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr bool	solve_multirhs (T TDLS_RESTRICT A, const int A_stride, int TDLS_RESTRICT piv, const int piv_stride, const T TDLS_RESTRICT b, T TDLS_RESTRICT x, const int rhs_stride, const int xcol_stride) noexcept
	Diagnostics-free solve_multirhs overload: no out-of-tile out-parameter at all.

template<bool internal_rhs, bool internal_piv, bool internal_matrix, bool oot_diagnostics = true>
TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr bool	solve_inplace (T TDLS_RESTRICT A, const int A_stride, int TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT y, const int rhs_stride, int &oot_count) noexcept
	Factorization with the forward substitution folded in.

template<bool internal_rhs, bool internal_piv, bool internal_matrix>
TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr bool	solve_inplace (T TDLS_RESTRICT A, const int A_stride, int TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT y, const int rhs_stride) noexcept
	Diagnostics-free solve_inplace overload: no out-of-tile out-parameter at all.

template<int nrhs, bool internal_rhs, bool internal_piv, bool internal_matrix, int pass_width = 0, bool oot_diagnostics = true>
TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr bool	solve_inplace_multirhs (T TDLS_RESTRICT A, const int A_stride, int TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT y, const int rhs_stride, const int xcol_stride, int &oot_count) noexcept
	factorize + substitute_inplace_multirhs in one call: y holds the nrhs unpermuted right-hand-side columns on entry and the nrhs solutions on exit.

template<int nrhs, bool internal_rhs, bool internal_piv, bool internal_matrix, int pass_width = 0>
TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr bool	solve_inplace_multirhs (T TDLS_RESTRICT A, const int A_stride, int TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT y, const int rhs_stride, const int xcol_stride) noexcept
	Diagnostics-free solve_inplace_multirhs overload: no out-of-tile out-parameter at all.

Static Public Attributes
static constexpr int	tile_size = Config.tile_size
	tile size (int)

static constexpr Schedule	schedule
	elimination schedule (RightLooking or LeftLooking)

static constexpr T	oot_threshold = Config.oot_threshold
	Acceptable-pivot threshold of the out-of-tile search, read once from the configuration (see TiledLUppConfig::oot_threshold).

static constexpr T	singular_floor = Config.singular_floor
	Singularity floor of the out-of-tile recovery, read once from the configuration (see TiledLUppConfig::singular_floor).

static constexpr int	full_tiles = N / tile_size

static constexpr int	last_tile_tail = N - full_tiles * tile_size
	Extent of the last, partial tile: 0 when N is a multiple of tile_size.

static constexpr int	num_tiles = full_tiles + (last_tile_tail > 0 ? 1 : 0)
	Tiles per dimension, the partial one included.

Detailed Description

template<typename T, int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>
struct tdls::TiledLUppSolverStatic< T, N, Config >

Tiled dense LU factorization with logical partial pivoting and out-of-tile pivot recovery, solving one NxN system per call.

All entry points are static, host- and device-callable, and take raw pointers pre-offset by the caller plus one runtime stride per array (see the file documentation for the addressing convention). Entry points:

factorize: A := P*L*U in place, pivots out (RL or LL)
substitute: x := U^-1 L^-1 P b (b and x distinct)
substitute_canonical: idem with b = e_col (tangent-operator columns)
substitute_inplace: idem with b == x (cycle-decomposition permute)
solve: factorize + substitute
solve_inplace: factorize with the forward pass folded in

Each substitution and solve entry point has a _multirhs twin taking nrhs right-hand-side columns per call (substitute_multirhs, substitute_canonical_multirhs, substitute_inplace_multirhs, solve_multirhs, solve_inplace_multirhs). The columns are solved in passes of pass_width columns (0 = one single pass); every L/U tile is loaded once per pass, and per-column results match nrhs single-column calls bitwise whatever the cutting. nrhs is a template parameter here and a runtime argument of the runtime solver, as the dimension itself; pass_width, a scheduling knob rather than a dimension, is a template parameter in both. It trails with a default of 0 here, so the common case needs no cutting argument at all. nrhs = 1 makes each twin collapse to its single-column form.

factorize/substitute stay separate so a factorization can be reused across several right-hand sides. Substitution is schedule-independent: RL and LL produce the same L\U layout.

The residency template booleans (internal_rhs / internal_piv / internal_matrix) deliberately have no default: they describe what the passed buffers ARE, so each call site must state them explicitly. A silent default could mismatch the actual argument and corrupt results without any diagnostic.

Template Parameters

T	scalar type (float, double or long double)
N	system dimension (N >= 1)
Config	compile-time knobs, passed as a constexpr value: tile size, schedule, pivoting thresholds, unroll policy; see TiledLUppConfig

Member Function Documentation

◆ bwd_only()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int pass_width, bool internal_rhs, bool internal_piv, bool internal_matrix>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::bwd_only	(	const T *TDLS_RESTRICT	A,
		const int	A_stride,
		const int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		T *TDLS_RESTRICT	x,
		const int	rhs_stride,
		const int	xcol_stride )

inlinestaticconstexprnoexcept

Backward pass alone, used by fwd_bwd and by solve_inplace (whose forward pass happens inside the factorization).

Template Parameters

pass_width	number of columns processed together
internal_rhs	residency of x
internal_piv	residency of piv
internal_matrix	residency of A

Parameters

[in]	A	factored matrix (caller-pre-offset)
[in]	A_stride	element stride of A (external mode)
[in]	piv	permutation from factorize
[in]	piv_stride	element stride of piv (external mode)
[in,out]	x	solution column(s)
[in]	rhs_stride	element stride of x (external mode)
[in]	xcol_stride	element stride between columns of x

◆ bwd_pull_one()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int k_extent, int m_extent, int pass_width, bool internal_rhs, bool internal_piv, bool internal_matrix>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::bwd_pull_one	(	const T *TDLS_RESTRICT	A,
		const int	A_stride,
		const int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		T *TDLS_RESTRICT	x,
		const int	rhs_stride,
		const int	xcol_stride,
		const int	k0,
		const int	m0 )

inlinestaticconstexprnoexcept

Backward pull: subtract U(k,m) * x_m from the x_k segment, for pass_width columns at once.

Template Parameters

k_extent	extent of the updated segment's tile
m_extent	extent of the trailing segment's tile
pass_width	number of columns processed together
internal_rhs	residency of x
internal_piv	residency of piv
internal_matrix	residency of A

Parameters

[in]	A	factored matrix (caller-pre-offset)
[in]	A_stride	element stride of A (external mode)
[in]	piv	permutation from factorize
[in]	piv_stride	element stride of piv (external mode)
[in,out]	x	solution column(s)
[in]	rhs_stride	element stride of x (external mode)
[in]	xcol_stride	element stride between columns of x
[in]	k0	first global row of the updated segment
[in]	m0	first global row of the trailing segment

◆ bwd_step()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int k_extent, int pass_width, bool internal_rhs, bool internal_piv, bool internal_matrix>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::bwd_step	(	const T *TDLS_RESTRICT	A,
		const int	A_stride,
		const int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		T *TDLS_RESTRICT	x,
		const int	rhs_stride,
		const int	xcol_stride,
		const int	k )

inlinestaticconstexprnoexcept

Backward step: pull the trailing contributions, then upper-solve the diagonal tile's segment.

Template Parameters

k_extent	extent of the diagonal tile of the step
pass_width	number of columns processed together
internal_rhs	residency of x
internal_piv	residency of piv
internal_matrix	residency of A

Parameters

[in]	A	factored matrix (caller-pre-offset)
[in]	A_stride	element stride of A (external mode)
[in]	piv	permutation from factorize
[in]	piv_stride	element stride of piv (external mode)
[in,out]	x	solution column(s)
[in]	rhs_stride	element stride of x (external mode)
[in]	xcol_stride	element stride between columns of x
[in]	k	step index (k0 = k*tile_size)

◆ factor_diag_column()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int k_extent, bool internal_piv, bool internal_matrix, bool oot_diagnostics, bool fuse_rhs, bool internal_rhs>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr bool tdls::TiledLUppSolverStatic< T, N, Config >::factor_diag_column	(	T *TDLS_RESTRICT	A,
		const int	A_stride,
		int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		const int	k0,
		T *TDLS_RESTRICT	tile,
		int &	oot_count,
		const int	c,
		T *TDLS_RESTRICT	y,
		const int	rhs_stride )

inlinestaticnodiscardconstexprnoexcept

One column step of the diagonal-tile factorization: pivot search (in-tile, then out-of-tile recovery), permutation update, row swap or cross-tile pull, column elimination.

tile holds the (LL: prior-corrected) KExKE diagonal tile; physical row swaps happen in the register tile only.

Template Parameters

k_extent	extent of the diagonal tile
internal_piv	residency of piv
internal_matrix	residency of A
oot_diagnostics	compile the out-of-tile counter in or out
fuse_rhs	apply the pivot swaps to the fused RHS y
internal_rhs	residency of y

Parameters

[in,out]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A (external mode)
[in,out]	piv	permutation (logical -> physical row)
[in]	piv_stride	element stride of piv (external mode)
[in]	k0	first global row/column of the tile
[in,out]	tile	register-resident diagonal tile
[in,out]	oot_count	out-of-tile search counter (oot_diagnostics)
[in]	c	tile column to factor
[in,out]	y	fused right-hand side (fuse_rhs only)
[in]	rhs_stride	element stride of y (external mode)

Returns: false when the matrix is singular at this column.

◆ factor_diag_tile()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int k_extent, bool internal_piv, bool internal_matrix, bool oot_diagnostics, bool fuse_rhs = false, bool internal_rhs = true>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr bool tdls::TiledLUppSolverStatic< T, N, Config >::factor_diag_tile	(	T *TDLS_RESTRICT	A,
		const int	A_stride,
		int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		const int	k0,
		T *TDLS_RESTRICT	tile,
		int &	oot_count,
		T *TDLS_RESTRICT	y = nullptr,
		const int	rhs_stride = 1 )

inlinestaticnodiscardconstexprnoexcept

Factor the KExKE diagonal tile in registers, with out-of-tile pivot recovery (drives the per-column loop of factor_diag_column).

Template Parameters

k_extent	extent of the diagonal tile
internal_piv	residency of piv
internal_matrix	residency of A
oot_diagnostics	compile the out-of-tile counter in or out
fuse_rhs	apply the pivot swaps to the fused RHS y
internal_rhs	residency of y

Parameters

[in,out]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A (external mode)
[in,out]	piv	permutation (logical -> physical row)
[in]	piv_stride	element stride of piv (external mode)
[in]	k0	first global row/column of the tile
[in,out]	tile	register-resident diagonal tile, loaded (and, LL: prior-corrected) by the caller
[in,out]	oot_count	out-of-tile search counter (oot_diagnostics)
[in,out]	y	fused right-hand side (fuse_rhs only)
[in]	rhs_stride	element stride of y (external mode)

Returns: false on a singular matrix.

◆ factorize() [1/2]

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<bool internal_piv, bool internal_matrix>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr bool tdls::TiledLUppSolverStatic< T, N, Config >::factorize	(	T *TDLS_RESTRICT	A,
		const int	A_stride,
		int *TDLS_RESTRICT	piv,
		const int	piv_stride )

inlinestaticnodiscardconstexprnoexcept

Diagnostics-free factorize overload: no out-of-tile out-parameter at all.

Template Parameters

internal_piv	residency of piv
internal_matrix	residency of A

Parameters

[in,out]	A	matrix, pre-offset by the caller
[in]	A_stride	element stride of A (external mode)
[out]	piv	permutation storage (always caller-provided)
[in]	piv_stride	element stride of piv (external mode)

Returns: false on a singular matrix.

◆ factorize() [2/2]

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<bool internal_piv, bool internal_matrix, bool oot_diagnostics = true, bool fuse_rhs = false, bool internal_rhs = true>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr bool tdls::TiledLUppSolverStatic< T, N, Config >::factorize	(	T *TDLS_RESTRICT	A,
		const int	A_stride,
		int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		int &	oot_count,
		T *TDLS_RESTRICT	y = nullptr,
		const int	rhs_stride = 1 )

inlinestaticnodiscardconstexprnoexcept

Factor A := P*L*U in place.

A becomes L\U (unit lower L below the diagonal, U above including it, the diagonal holding the pivot RECIPROCALS), under logical row permutation piv (piv[i] = physical row holding logical row i).

Template Parameters

internal_piv	true: piv is a caller-local int[N], stride ignored; false: piv is remote, indexed with piv_stride
internal_matrix	true: A is a caller-local T[N*N], stride ignored; false: A is remote, indexed with A_stride
oot_diagnostics	when false, the out-of-tile diagnostics are compiled out entirely (no counter register, no increments, oot_count untouched); use the overload without the out-parameter

Parameters

[in,out]	A	matrix, pre-offset by the caller
[in]	A_stride	element stride of A (external mode)
[out]	piv	permutation storage (always caller-provided)
[in]	piv_stride	element stride of piv (external mode)

Template Parameters

fuse_rhs	internal hook of solve_inplace: folds the forward substitution of y into the factorization
internal_rhs	residency of the fused y (meaningful with fuse_rhs only)

Parameters

[out]	oot_count	number of columns whose best in-tile pivot fell below oot_threshold
[in,out]	y	fused right-hand side (fuse_rhs only)
[in]	rhs_stride	element stride of y (external mode)

Returns: false on a singular matrix.

◆ fwd_bwd()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int pass_width, bool internal_rhs, bool internal_piv, bool internal_matrix>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::fwd_bwd	(	const T *TDLS_RESTRICT	A,
		const int	A_stride,
		const int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		T *TDLS_RESTRICT	x,
		const int	rhs_stride,
		const int	xcol_stride )

inlinestaticconstexprnoexcept

Triangular solves on already-permuted column(s) x. pass_width columns are processed per tile visit, so every L/U tile is loaded once for the whole block instead of once per column (pass_width = 1 is the single-RHS case).

Template Parameters

pass_width	number of columns processed together
internal_rhs	residency of x
internal_piv	residency of piv
internal_matrix	residency of A

Parameters

[in]	A	factored matrix (caller-pre-offset)
[in]	A_stride	element stride of A (external mode)
[in]	piv	permutation from factorize
[in]	piv_stride	element stride of piv (external mode)
[in,out]	x	solution column(s)
[in]	rhs_stride	element stride of x (external mode)
[in]	xcol_stride	element stride between columns of x

◆ fwd_push_one()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int k_extent, int m_extent, int pass_width, bool internal_rhs, bool internal_piv, bool internal_matrix>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::fwd_push_one	(	const T *TDLS_RESTRICT	A,
		const int	A_stride,
		const int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		T *TDLS_RESTRICT	x,
		const int	rhs_stride,
		const int	xcol_stride,
		const int	k0,
		const int	m0 )

inlinestaticconstexprnoexcept

Forward push: subtract L(m,k) * x_k from the x_m segment, for pass_width columns at once.

Template Parameters

k_extent	extent of the solved segment's tile
m_extent	extent of the target segment's tile
pass_width	number of columns processed together
internal_rhs	residency of x
internal_piv	residency of piv
internal_matrix	residency of A

Parameters

[in]	A	factored matrix (caller-pre-offset)
[in]	A_stride	element stride of A (external mode)
[in]	piv	permutation from factorize
[in]	piv_stride	element stride of piv (external mode)
[in,out]	x	solution column(s)
[in]	rhs_stride	element stride of x (external mode)
[in]	xcol_stride	element stride between columns of x
[in]	k0	first global row of the solved segment
[in]	m0	first global row of the target segment

◆ fwd_step()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int k_extent, int pass_width, bool internal_rhs, bool internal_piv, bool internal_matrix>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::fwd_step	(	const T *TDLS_RESTRICT	A,
		const int	A_stride,
		const int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		T *TDLS_RESTRICT	x,
		const int	rhs_stride,
		const int	xcol_stride,
		const int	k )

inlinestaticconstexprnoexcept

Forward step: unit-lower solve the diagonal tile's segment, then push it into the tiles below.

Template Parameters

k_extent	extent of the diagonal tile of the step
pass_width	number of columns processed together
internal_rhs	residency of x
internal_piv	residency of piv
internal_matrix	residency of A

Parameters

[in]	A	factored matrix (caller-pre-offset)
[in]	A_stride	element stride of A (external mode)
[in]	piv	permutation from factorize
[in]	piv_stride	element stride of piv (external mode)
[in,out]	x	solution column(s)
[in]	rhs_stride	element stride of x (external mode)
[in]	xcol_stride	element stride between columns of x
[in]	k	step index (k0 = k*tile_size)

◆ ll_correct_tile()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int row_extent, int col_extent, bool internal_piv, bool internal_matrix>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::ll_correct_tile	(	const T *TDLS_RESTRICT	A,
		const int	A_stride,
		const int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		const int	row0,
		const int	col0,
		const int	k0,
		T *TDLS_RESTRICT	t )

inlinestaticconstexprnoexcept

LL: t (RExCE, rows row0.., cols col0..) -= sum over prior tiles bj < k0/tile_size of L(row0.., bj) * U(bj, col0..). All prior tiles are full.

Template Parameters

row_extent	row extent of the corrected tile
col_extent	column extent of the corrected tile
internal_piv	residency of piv
internal_matrix	residency of A

Parameters

[in]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A (external mode)
[in]	piv	permutation (logical -> physical row)
[in]	piv_stride	element stride of piv (external mode)
[in]	row0	first global row of the corrected tile
[in]	col0	first global column of the corrected tile
[in]	k0	first row/column of the current step
[in,out]	t	register tile being corrected

◆ ll_step()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int k_extent, bool internal_piv, bool internal_matrix, bool oot_diagnostics, bool fuse_rhs = false, bool internal_rhs = true>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr bool tdls::TiledLUppSolverStatic< T, N, Config >::ll_step	(	T *TDLS_RESTRICT	A,
		const int	A_stride,
		int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		const int	k,
		int &	oot_count,
		T *TDLS_RESTRICT	y = nullptr,
		const int	rhs_stride = 1 )

inlinestaticnodiscardconstexprnoexcept

LL: one full factorization step (correct + factor the diagonal tile, then its L and U panels).

Template Parameters

k_extent	extent of the diagonal tile of this step
internal_piv	residency of piv
internal_matrix	residency of A
oot_diagnostics	compile the out-of-tile counter in or out
fuse_rhs	apply the step to the fused RHS y (solve_inplace)
internal_rhs	residency of y

Parameters

[in,out]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A (external mode)
[in,out]	piv	permutation (logical -> physical row)
[in]	piv_stride	element stride of piv (external mode)
[in]	k	step index (k0 = k*tile_size)
[in,out]	oot_count	out-of-tile search counter (oot_diagnostics)
[in,out]	y	fused right-hand side (fuse_rhs only)
[in]	rhs_stride	element stride of y (external mode)

Returns: false on a singular matrix.

◆ ll_update_below_one()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int k_extent, int i_extent, bool internal_piv, bool internal_matrix, bool fuse_rhs = false, bool internal_rhs = true>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::ll_update_below_one	(	T *TDLS_RESTRICT	A,
		const int	A_stride,
		const int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		const T *TDLS_RESTRICT	tile,
		const int	k0,
		const int	i0,
		T *TDLS_RESTRICT	y = nullptr,
		const int	rhs_stride = 1 )

inlinestaticconstexprnoexcept

LL: correct + TRSM one L-panel tile below the diagonal, with the optional fused forward-substitution push.

Template Parameters

k_extent	extent of the diagonal tile
i_extent	row extent of the updated tile
internal_piv	residency of piv
internal_matrix	residency of A
fuse_rhs	push the fused RHS y along (solve_inplace)
internal_rhs	residency of y

Parameters

[in,out]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A (external mode)
[in]	piv	permutation (logical -> physical row)
[in]	piv_stride	element stride of piv (external mode)
[in]	tile	factored diagonal tile
[in]	k0	first global row/column of the step
[in]	i0	first global row of the updated tile
[in,out]	y	fused right-hand side (fuse_rhs only)
[in]	rhs_stride	element stride of y (external mode)

◆ ll_update_right_one()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int k_extent, int j_extent, bool internal_piv, bool internal_matrix>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::ll_update_right_one	(	T *TDLS_RESTRICT	A,
		const int	A_stride,
		const int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		const T *TDLS_RESTRICT	tile,
		const int	k0,
		const int	j0 )

inlinestaticconstexprnoexcept

LL: correct + TRSM one U-panel tile right of the diagonal.

Template Parameters

k_extent	extent of the diagonal tile
j_extent	column extent of the updated tile
internal_piv	residency of piv
internal_matrix	residency of A

Parameters

[in,out]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A (external mode)
[in]	piv	permutation (logical -> physical row)
[in]	piv_stride	element stride of piv (external mode)
[in]	tile	factored diagonal tile
[in]	k0	first global row/column of the step
[in]	j0	first global column of the updated tile

◆ load_tile()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int row_extent, int col_extent, bool internal_matrix>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::load_tile	(	const T *TDLS_RESTRICT	A,
		const int	A_stride,
		const int *TDLS_RESTRICT	prow,
		const int	col0,
		T *TDLS_RESTRICT	t )

inlinestaticconstexprnoexcept

Load an RxC tile through a cached physical-row segment.

Template Parameters

row_extent	tile row extent
col_extent	tile column extent
internal_matrix	residency of A

Parameters

[in]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A (external mode)
[in]	prow	physical rows of the tile
[in]	col0	first global column of the tile
[out]	t	destination register tile

◆ load_tile_piv()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int row_extent, int col_extent, bool internal_piv, bool internal_matrix>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::load_tile_piv	(	const T *TDLS_RESTRICT	A,
		const int	A_stride,
		const int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		const int	row0,
		const int	col0,
		T *TDLS_RESTRICT	t )

inlinestaticconstexprnoexcept

Load an RxC tile, reading the permutation inline (one read per row).

Template Parameters

row_extent	tile row extent
col_extent	tile column extent
internal_piv	residency of piv
internal_matrix	residency of A

Parameters

[in]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A (external mode)
[in]	piv	permutation (logical -> physical row)
[in]	piv_stride	element stride of piv (external mode)
[in]	row0	first global (logical) row of the tile
[in]	col0	first global column of the tile
[out]	t	destination register tile

◆ load_tile_piv_lower()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int row_extent, bool internal_piv, bool internal_matrix>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::load_tile_piv_lower	(	const T *TDLS_RESTRICT	A,
		const int	A_stride,
		const int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		const int	row0,
		const int	col0,
		T *TDLS_RESTRICT	t )

inlinestaticconstexprnoexcept

Triangular variant of the diagonal-tile load for the forward substitution: only the strict lower triangle of L\U is read.

The untouched slots of the register tile are never read: compile-time loop bounds, no branches, results bitwise identical.

Template Parameters

row_extent	tile extent
internal_piv	residency of piv
internal_matrix	residency of A

Parameters

[in]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A (external mode)
[in]	piv	permutation (logical -> physical row)
[in]	piv_stride	element stride of piv (external mode)
[in]	row0	first global (logical) row of the tile
[in]	col0	first global column of the tile
[out]	t	destination register tile

◆ load_tile_piv_upper()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int row_extent, bool internal_piv, bool internal_matrix>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::load_tile_piv_upper	(	const T *TDLS_RESTRICT	A,
		const int	A_stride,
		const int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		const int	row0,
		const int	col0,
		T *TDLS_RESTRICT	t )

inlinestaticconstexprnoexcept

Triangular variant of the diagonal-tile load for the backward substitution: only the upper triangle including the diagonal is read.

Template Parameters

row_extent	tile extent
internal_piv	residency of piv
internal_matrix	residency of A

Parameters

[in]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A (external mode)
[in]	piv	permutation (logical -> physical row)
[in]	piv_stride	element stride of piv (external mode)
[in]	row0	first global (logical) row of the tile
[in]	col0	first global column of the tile
[out]	t	destination register tile

◆ rl_schur_one()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int k_extent, int i_extent, int j_extent, bool internal_matrix>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::rl_schur_one	(	T *TDLS_RESTRICT	A,
		const int	A_stride,
		const int *TDLS_RESTRICT	pk,
		const int *TDLS_RESTRICT	pi,
		const T *TDLS_RESTRICT	Aik,
		const int	j0 )

inlinestaticconstexprnoexcept

RL: one Schur-complement tile update, Aij -= Aik * Akj, streaming the factored Akj row by row from remote memory (it is not worth a third register tile).

Template Parameters

k_extent	inner extent (current step)
i_extent	row extent of the updated tile
j_extent	column extent of the updated tile
internal_matrix	residency of A

Parameters

[in,out]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A (external mode)
[in]	pk	physical rows of the factored block row
[in]	pi	physical rows of the updated block row
[in]	Aik	register L panel of the updated block row
[in]	j0	first global column of the updated tile

◆ rl_step()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int k_extent, bool internal_piv, bool internal_matrix, bool oot_diagnostics, bool fuse_rhs = false, bool internal_rhs = true>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr bool tdls::TiledLUppSolverStatic< T, N, Config >::rl_step	(	T *TDLS_RESTRICT	A,
		const int	A_stride,
		int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		const int	k,
		int &	oot_count,
		T *TDLS_RESTRICT	y = nullptr,
		const int	rhs_stride = 1 )

inlinestaticnodiscardconstexprnoexcept

RL: one full factorization step (diagonal tile + trailing updates).

Template Parameters

k_extent	extent of the diagonal tile of this step
internal_piv	residency of piv
internal_matrix	residency of A
oot_diagnostics	compile the out-of-tile counter in or out
fuse_rhs	apply the step to the fused RHS y (solve_inplace)
internal_rhs	residency of y

Parameters

[in,out]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A (external mode)
[in,out]	piv	permutation (logical -> physical row)
[in]	piv_stride	element stride of piv (external mode)
[in]	k	step index (k0 = k*tile_size)
[in,out]	oot_count	out-of-tile search counter (oot_diagnostics)
[in,out]	y	fused right-hand side (fuse_rhs only)
[in]	rhs_stride	element stride of y (external mode)

Returns: false on a singular matrix.

◆ rl_trsm_right_one()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int k_extent, int j_extent, bool internal_matrix>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::rl_trsm_right_one	(	T *TDLS_RESTRICT	A,
		const int	A_stride,
		const int *TDLS_RESTRICT	pk,
		const T *TDLS_RESTRICT	tile,
		const int	j0 )

inlinestaticconstexprnoexcept

RL: one row-panel tile update, Akj := L^-1 Akj.

Template Parameters

k_extent	extent of the factored diagonal tile
j_extent	column extent of the updated tile
internal_matrix	residency of A

Parameters

[in,out]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A (external mode)
[in]	pk	physical rows of the diagonal block row
[in]	tile	factored diagonal tile
[in]	j0	first global column of the updated tile

◆ rl_update_row_one()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int k_extent, int i_extent, bool internal_piv, bool internal_matrix, bool fuse_rhs = false, bool internal_rhs = true>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::rl_update_row_one	(	T *TDLS_RESTRICT	A,
		const int	A_stride,
		const int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		const int *TDLS_RESTRICT	pk,
		const T *TDLS_RESTRICT	tile,
		const int	k,
		const int	i0,
		T *TDLS_RESTRICT	y = nullptr,
		const int	rhs_stride = 1 )

inlinestaticconstexprnoexcept

RL: TRSM down + Schur sweep of one row block below the diagonal, with the optional fused forward-substitution push.

Template Parameters

k_extent	extent of the diagonal tile
i_extent	row extent of the updated block row
internal_piv	residency of piv
internal_matrix	residency of A
fuse_rhs	push the fused RHS y along (solve_inplace)
internal_rhs	residency of y

Parameters

[in,out]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A (external mode)
[in]	piv	permutation (logical -> physical row)
[in]	piv_stride	element stride of piv (external mode)
[in]	pk	physical rows of the diagonal tile
[in]	tile	factored diagonal tile
[in]	k	step index (k0 = k*tile_size)
[in]	i0	first global row of the updated block row
[in,out]	y	fused right-hand side (fuse_rhs only)
[in]	rhs_stride	element stride of y (external mode)

◆ solve() [1/2]

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<bool internal_rhs, bool internal_piv, bool internal_matrix>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr bool tdls::TiledLUppSolverStatic< T, N, Config >::solve	(	T *TDLS_RESTRICT	A,
		const int	A_stride,
		int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		const T *TDLS_RESTRICT	b,
		T *TDLS_RESTRICT	x,
		const int	rhs_stride )

inlinestaticnodiscardconstexprnoexcept

Diagnostics-free solve overload: no out-of-tile out-parameter at all.

Template Parameters

internal_rhs	residency of b and x
internal_piv	residency of piv
internal_matrix	residency of A

Parameters

[in,out]	A	on entry the matrix (pre-offset by the caller), on exit its factorization
[in]	A_stride	element stride of A (external mode)
[out]	piv	permutation storage (always caller-provided)
[in]	piv_stride	element stride of piv (external mode)
[in]	b	right-hand side, in original order
[out]	x	solution
[in]	rhs_stride	element stride of b and x (external mode)

Returns: false on a singular matrix.

◆ solve() [2/2]

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<bool internal_rhs, bool internal_piv, bool internal_matrix, bool oot_diagnostics = true>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr bool tdls::TiledLUppSolverStatic< T, N, Config >::solve	(	T *TDLS_RESTRICT	A,
		const int	A_stride,
		int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		const T *TDLS_RESTRICT	b,
		T *TDLS_RESTRICT	x,
		const int	rhs_stride,
		int &	oot_count )

inlinestaticnodiscardconstexprnoexcept

factorize + substitute in one call.

Template Parameters

internal_rhs	residency of b and x
internal_piv	residency of piv
internal_matrix	residency of A
oot_diagnostics	compile the out-of-tile counter in or out

Parameters

[in,out]	A	on entry the matrix (pre-offset by the caller), on exit its factorization (usable for further substitute* calls)
[in]	A_stride	element stride of A (external mode)
[out]	piv	permutation storage (always caller-provided)
[in]	piv_stride	element stride of piv (external mode)
[in]	b	right-hand side, in original order
[out]	x	solution
[in]	rhs_stride	element stride of b and x (external mode)
[out]	oot_count	number of columns whose best in-tile pivot fell below oot_threshold

Returns: false on a singular matrix.

◆ solve_inplace() [1/2]

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<bool internal_rhs, bool internal_piv, bool internal_matrix>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr bool tdls::TiledLUppSolverStatic< T, N, Config >::solve_inplace	(	T *TDLS_RESTRICT	A,
		const int	A_stride,
		int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		T *TDLS_RESTRICT	y,
		const int	rhs_stride )

inlinestaticnodiscardconstexprnoexcept

Diagnostics-free solve_inplace overload: no out-of-tile out-parameter at all.

Template Parameters

internal_rhs	residency of y
internal_piv	residency of piv
internal_matrix	residency of A

Parameters

[in,out]	A	on entry the matrix (pre-offset by the caller), on exit its factorization
[in]	A_stride	element stride of A (external mode)
[out]	piv	permutation storage (always caller-provided)
[in]	piv_stride	element stride of piv (external mode)
[in,out]	y	right-hand side on entry, solution on exit
[in]	rhs_stride	element stride of y (external mode)

Returns: false on a singular matrix (y left partially updated).

◆ solve_inplace() [2/2]

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<bool internal_rhs, bool internal_piv, bool internal_matrix, bool oot_diagnostics = true>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr bool tdls::TiledLUppSolverStatic< T, N, Config >::solve_inplace	(	T *TDLS_RESTRICT	A,
		const int	A_stride,
		int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		T *TDLS_RESTRICT	y,
		const int	rhs_stride,
		int &	oot_count )

inlinestaticnodiscardconstexprnoexcept

Factorization with the forward substitution folded in.

y holds the unpermuted RHS on entry and the solution on exit. Each L panel is applied to y while it sits in registers during the factorization, so the separate forward pass disappears, and all its tile reloads with it; only the backward pass remains. y follows the rows through pivoting (swap hook in factor_diag_column), which makes the result bitwise-identical to factorize + substitute_inplace.

Template Parameters

internal_rhs	residency of y
internal_piv	residency of piv
internal_matrix	residency of A
oot_diagnostics	compile the out-of-tile counter in or out

Parameters

[in,out]	A	on entry the matrix (pre-offset by the caller), on exit its factorization
[in]	A_stride	element stride of A (external mode)
[out]	piv	permutation storage (always caller-provided)
[in]	piv_stride	element stride of piv (external mode)
[in,out]	y	right-hand side on entry, solution on exit
[in]	rhs_stride	element stride of y (external mode)
[out]	oot_count	number of columns whose best in-tile pivot fell below oot_threshold

Returns: false on a singular matrix (y left partially updated).

◆ solve_inplace_multirhs() [1/2]

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int nrhs, bool internal_rhs, bool internal_piv, bool internal_matrix, int pass_width = 0>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr bool tdls::TiledLUppSolverStatic< T, N, Config >::solve_inplace_multirhs	(	T *TDLS_RESTRICT	A,
		const int	A_stride,
		int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		T *TDLS_RESTRICT	y,
		const int	rhs_stride,
		const int	xcol_stride )

inlinestaticnodiscardconstexprnoexcept

Diagnostics-free solve_inplace_multirhs overload: no out-of-tile out-parameter at all.

Template Parameters

nrhs	total number of right-hand-side columns
internal_rhs	residency of y
internal_piv	residency of piv
internal_matrix	residency of A
pass_width	columns solved together per substitution pass; the default 0, or any value >= nrhs, means one single pass

Parameters

[in,out]	A	on entry the matrix (pre-offset by the caller), on exit its factorization
[in]	A_stride	element stride of A (external mode)
[out]	piv	permutation storage (always caller-provided)
[in]	piv_stride	element stride of piv (external mode)
[in,out]	y	nrhs right-hand-side columns on entry, nrhs solution columns on exit
[in]	rhs_stride	element stride of y (external mode)
[in]	xcol_stride	element stride between columns of y (external mode)

Returns: false on a singular matrix (y left untouched).

◆ solve_inplace_multirhs() [2/2]

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int nrhs, bool internal_rhs, bool internal_piv, bool internal_matrix, int pass_width = 0, bool oot_diagnostics = true>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr bool tdls::TiledLUppSolverStatic< T, N, Config >::solve_inplace_multirhs	(	T *TDLS_RESTRICT	A,
		const int	A_stride,
		int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		T *TDLS_RESTRICT	y,
		const int	rhs_stride,
		const int	xcol_stride,
		int &	oot_count )

inlinestaticnodiscardconstexprnoexcept

factorize + substitute_inplace_multirhs in one call: y holds the nrhs unpermuted right-hand-side columns on entry and the nrhs solutions on exit.

Unlike solve_inplace, the forward pass is not folded into the factorization: a pass amortizes the forward tile loads across its columns, which removes most of what the folding saves.

Template Parameters

nrhs	total number of right-hand-side columns
internal_rhs	residency of y
internal_piv	residency of piv
internal_matrix	residency of A
pass_width	columns solved together per substitution pass; the default 0, or any value >= nrhs, means one single pass
oot_diagnostics	compile the out-of-tile counter in or out

Parameters

[in,out]	A	on entry the matrix (pre-offset by the caller), on exit its factorization
[in]	A_stride	element stride of A (external mode)
[out]	piv	permutation storage (always caller-provided)
[in]	piv_stride	element stride of piv (external mode)
[in,out]	y	nrhs right-hand-side columns on entry, nrhs solution columns on exit
[in]	rhs_stride	element stride of y (external mode)
[in]	xcol_stride	element stride between columns of y (external mode)
[out]	oot_count	number of columns whose best in-tile pivot fell below oot_threshold

Returns: false on a singular matrix (y left untouched).

◆ solve_multirhs() [1/2]

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int nrhs, bool internal_rhs, bool internal_piv, bool internal_matrix, int pass_width = 0>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr bool tdls::TiledLUppSolverStatic< T, N, Config >::solve_multirhs	(	T *TDLS_RESTRICT	A,
		const int	A_stride,
		int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		const T *TDLS_RESTRICT	b,
		T *TDLS_RESTRICT	x,
		const int	rhs_stride,
		const int	xcol_stride )

inlinestaticnodiscardconstexprnoexcept

Diagnostics-free solve_multirhs overload: no out-of-tile out-parameter at all.

Template Parameters

nrhs	total number of right-hand-side columns
internal_rhs	residency of b and x
internal_piv	residency of piv
internal_matrix	residency of A
pass_width	columns solved together per substitution pass; the default 0, or any value >= nrhs, means one single pass

Parameters

[in,out]	A	on entry the matrix (pre-offset by the caller), on exit its factorization
[in]	A_stride	element stride of A (external mode)
[out]	piv	permutation storage (always caller-provided)
[in]	piv_stride	element stride of piv (external mode)
[in]	b	nrhs right-hand-side columns, in original order
[out]	x	nrhs solution columns
[in]	rhs_stride	element stride of b and x (external mode)
[in]	xcol_stride	element stride between columns of b and x (external mode)

Returns: false on a singular matrix.

◆ solve_multirhs() [2/2]

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int nrhs, bool internal_rhs, bool internal_piv, bool internal_matrix, int pass_width = 0, bool oot_diagnostics = true>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr bool tdls::TiledLUppSolverStatic< T, N, Config >::solve_multirhs	(	T *TDLS_RESTRICT	A,
		const int	A_stride,
		int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		const T *TDLS_RESTRICT	b,
		T *TDLS_RESTRICT	x,
		const int	rhs_stride,
		const int	xcol_stride,
		int &	oot_count )

inlinestaticnodiscardconstexprnoexcept

factorize + substitute_multirhs in one call.

Template Parameters

nrhs	total number of right-hand-side columns
internal_rhs	residency of b and x
internal_piv	residency of piv
internal_matrix	residency of A
pass_width	columns solved together per substitution pass; the default 0, or any value >= nrhs, means one single pass
oot_diagnostics	compile the out-of-tile counter in or out

Parameters

[in,out]	A	on entry the matrix (pre-offset by the caller), on exit its factorization (usable for further substitute* calls)
[in]	A_stride	element stride of A (external mode)
[out]	piv	permutation storage (always caller-provided)
[in]	piv_stride	element stride of piv (external mode)
[in]	b	nrhs right-hand-side columns, in original order
[out]	x	nrhs solution columns
[in]	rhs_stride	element stride of b and x (external mode)
[in]	xcol_stride	element stride between columns of b and x (external mode)
[out]	oot_count	number of columns whose best in-tile pivot fell below oot_threshold

Returns: false on a singular matrix.

◆ store_tile()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int row_extent, int col_extent, bool internal_matrix>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::store_tile	(	T *TDLS_RESTRICT	A,
		const int	A_stride,
		const int *TDLS_RESTRICT	prow,
		const int	col0,
		const T *TDLS_RESTRICT	t )

inlinestaticconstexprnoexcept

Store an RxC tile through a cached physical-row segment.

Template Parameters

row_extent	tile row extent
col_extent	tile column extent
internal_matrix	residency of A

Parameters

[in,out]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A (external mode)
[in]	prow	physical rows of the tile
[in]	col0	first global column of the tile
[in]	t	source register tile

◆ store_tile_piv()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int row_extent, int col_extent, bool internal_piv, bool internal_matrix>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::store_tile_piv	(	T *TDLS_RESTRICT	A,
		const int	A_stride,
		const int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		const int	row0,
		const int	col0,
		const T *TDLS_RESTRICT	t )

inlinestaticconstexprnoexcept

Store an RxC tile, reading the permutation inline (one read per row).

Template Parameters

row_extent	tile row extent
col_extent	tile column extent
internal_piv	residency of piv
internal_matrix	residency of A

Parameters

[in,out]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A (external mode)
[in]	piv	permutation (logical -> physical row)
[in]	piv_stride	element stride of piv (external mode)
[in]	row0	first global (logical) row of the tile
[in]	col0	first global column of the tile
[in]	t	source register tile

◆ substitute()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<bool internal_rhs, bool internal_piv, bool internal_matrix>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::substitute	(	const T *TDLS_RESTRICT	A,
		const int	A_stride,
		const int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		const T *TDLS_RESTRICT	b,
		T *TDLS_RESTRICT	x,
		const int	rhs_stride )

inlinestaticconstexprnoexcept

Solve x := U^-1 L^-1 P b from a prior factorize. b and x must not alias (use substitute_inplace for the aliased single-buffer pattern).

Template Parameters

internal_rhs	true: b and x are caller-local T[N], strides ignored; false: remote, indexed with rhs_stride
internal_piv	residency of piv
internal_matrix	residency of A

Parameters

[in]	A	factored matrix produced by factorize
[in]	A_stride	element stride of A (external mode)
[in]	piv	permutation produced by factorize
[in]	piv_stride	element stride of piv (external mode)
[in]	b	right-hand side, in original (unpermuted) order
[out]	x	solution
[in]	rhs_stride	element stride of b and x (external mode)

◆ substitute_canonical()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<bool internal_rhs, bool internal_piv, bool internal_matrix>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::substitute_canonical	(	const T *TDLS_RESTRICT	A,
		const int	A_stride,
		const int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		const int	col,
		T *TDLS_RESTRICT	x,
		const int	rhs_stride )

inlinestaticconstexprnoexcept

Solve with b = e_col generated on the fly: the consistent-tangent-operator path.

Thin alias of the canonical pass engine below with pass_width = 1, which collapses to exactly the single-column code, so there is no separate implementation to maintain.

Template Parameters

internal_rhs	residency of x
internal_piv	residency of piv
internal_matrix	residency of A

Parameters

[in]	A	factored matrix produced by factorize
[in]	A_stride	element stride of A (external mode)
[in]	piv	permutation produced by factorize
[in]	piv_stride	element stride of piv (external mode)
[in]	col	index of the canonical column e_col
[out]	x	solution
[in]	rhs_stride	element stride of x (external mode)

◆ substitute_canonical_multirhs()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int nrhs, bool internal_rhs, bool internal_piv, bool internal_matrix, int pass_width = 0>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::substitute_canonical_multirhs	(	const T *TDLS_RESTRICT	A,
		const int	A_stride,
		const int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		const int	col0,
		T *TDLS_RESTRICT	x,
		const int	rhs_stride,
		const int	xcol_stride )

inlinestaticconstexprnoexcept

nrhs canonical columns e_col0 .. e_{col0+nrhs-1} solved from a prior factorize: the columns of the inverse that a consistent tangent operator needs.

Column w lives at x + w*N (internal) or x + w*xcol_stride (remote). The columns are solved in passes of pass_width columns; within a pass, every L/U tile is loaded once for all the columns of the pass. Per-column arithmetic is identical whatever the cutting, and identical to nrhs separate substitute_canonical calls: results match them bitwise (nrhs = 1 collapses to substitute_canonical exactly).

Template Parameters

nrhs	number of consecutive canonical columns
internal_rhs	residency of x
internal_piv	residency of piv
internal_matrix	residency of A
pass_width	columns solved together per pass; the default 0, or any value >= nrhs, means one single pass of nrhs columns

Parameters

[in]	A	factored matrix produced by factorize
[in]	A_stride	element stride of A (external mode)
[in]	piv	permutation produced by factorize
[in]	piv_stride	element stride of piv (external mode)
[in]	col0	index of the first canonical column
[out]	x	nrhs solution columns
[in]	rhs_stride	element stride of x (external mode)
[in]	xcol_stride	element stride between columns of x (external mode)

◆ substitute_canonical_multirhs_pass()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int pass_width, bool internal_rhs, bool internal_piv, bool internal_matrix>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::substitute_canonical_multirhs_pass	(	const T *TDLS_RESTRICT	A,
		const int	A_stride,
		const int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		const int	col0,
		T *TDLS_RESTRICT	x,
		const int	rhs_stride,
		const int	xcol_stride )

inlinestaticconstexprnoexcept

One pass of the canonical multi right-hand-side substitution: pass_width canonical columns e_col0 .. e_{col0+pass_width-1} generated in permuted order into x, then one triangular sweep for all pass_width columns together. Internal engine of substitute_canonical_multirhs.

Template Parameters

pass_width	number of columns of this pass
internal_rhs	residency of x
internal_piv	residency of piv
internal_matrix	residency of A

Parameters

[in]	A	factored matrix produced by factorize
[in]	A_stride	element stride of A (external mode)
[in]	piv	permutation produced by factorize
[in]	piv_stride	element stride of piv (external mode)
[in]	col0	index of the first canonical column
[out]	x	pass_width solution columns
[in]	rhs_stride	element stride of x (external mode)
[in]	xcol_stride	element stride between columns of x (external mode)

◆ substitute_inplace()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<bool internal_rhs, bool internal_piv, bool internal_matrix>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::substitute_inplace	(	const T *TDLS_RESTRICT	A,
		const int	A_stride,
		const int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		T *TDLS_RESTRICT	x,
		const int	rhs_stride )

inlinestaticconstexprnoexcept

Solve in place: x already holds the unpermuted RHS on entry and the solution on exit.

The permutation is applied in place by cycle decomposition, each cycle rotated once from its smallest index. Up to N = 64 a bitmask tracks the visited rows (a single 32- or 64-bit register); beyond, a cycle-leader scan is used instead (zero storage, an extra integer walk of each orbit). Both paths move the same values in the same order.

Template Parameters

internal_rhs	residency of x
internal_piv	residency of piv
internal_matrix	residency of A

Parameters

[in]	A	factored matrix produced by factorize
[in]	A_stride	element stride of A (external mode)
[in]	piv	permutation produced by factorize
[in]	piv_stride	element stride of piv (external mode)
[in,out]	x	right-hand side on entry, solution on exit
[in]	rhs_stride	element stride of x (external mode)

◆ substitute_inplace_multirhs()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int nrhs, bool internal_rhs, bool internal_piv, bool internal_matrix, int pass_width = 0>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::substitute_inplace_multirhs	(	const T *TDLS_RESTRICT	A,
		const int	A_stride,
		const int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		T *TDLS_RESTRICT	x,
		const int	rhs_stride,
		const int	xcol_stride )

inlinestaticconstexprnoexcept

nrhs columns solved in place: x holds the nrhs unpermuted right-hand-side columns on entry and the nrhs solutions on exit.

The permutation is applied by the same cycle decomposition as substitute_inplace. The columns are solved in passes of pass_width columns; within a pass, every L/U tile is loaded once for all the columns of the pass. Per-column values are identical whatever the cutting, and match nrhs separate substitute_inplace calls bitwise (nrhs = 1 collapses to substitute_inplace exactly).

Template Parameters

nrhs	total number of right-hand-side columns
internal_rhs	residency of x
internal_piv	residency of piv
internal_matrix	residency of A
pass_width	columns solved together per pass; the default 0, or any value >= nrhs, means one single pass of nrhs columns

Parameters

[in]	A	factored matrix produced by factorize
[in]	A_stride	element stride of A (external mode)
[in]	piv	permutation produced by factorize
[in]	piv_stride	element stride of piv (external mode)
[in,out]	x	nrhs right-hand-side columns on entry, nrhs solution columns on exit
[in]	rhs_stride	element stride of x (external mode)
[in]	xcol_stride	element stride between columns of x (external mode)

◆ substitute_inplace_multirhs_pass()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int pass_width, bool internal_rhs, bool internal_piv, bool internal_matrix>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::substitute_inplace_multirhs_pass	(	const T *TDLS_RESTRICT	A,
		const int	A_stride,
		const int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		T *TDLS_RESTRICT	x,
		const int	rhs_stride,
		const int	xcol_stride )

inlinestaticconstexprnoexcept

One pass of the in-place multi right-hand-side substitution: pass_width columns of x permuted by the cycle decomposition of substitute_inplace (each move carrying the pass_width column entries of its row), then one triangular sweep for all pass_width columns together. Internal engine of substitute_inplace_multirhs.

Template Parameters

pass_width	number of columns of this pass
internal_rhs	residency of x
internal_piv	residency of piv
internal_matrix	residency of A

Parameters

[in]	A	factored matrix produced by factorize
[in]	A_stride	element stride of A (external mode)
[in]	piv	permutation produced by factorize
[in]	piv_stride	element stride of piv (external mode)
[in,out]	x	pass_width right-hand-side columns on entry, pass_width solution columns on exit
[in]	rhs_stride	element stride of x (external mode)
[in]	xcol_stride	element stride between columns of x (external mode)

◆ substitute_multirhs()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int nrhs, bool internal_rhs, bool internal_piv, bool internal_matrix, int pass_width = 0>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::substitute_multirhs	(	const T *TDLS_RESTRICT	A,
		const int	A_stride,
		const int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		const T *TDLS_RESTRICT	b,
		T *TDLS_RESTRICT	x,
		const int	rhs_stride,
		const int	xcol_stride )

inlinestaticconstexprnoexcept

Solve nrhs right-hand-side columns from a prior factorize: X := U^-1 L^-1 P B, column by column. b and x must not alias (use substitute_inplace_multirhs for the aliased single-buffer pattern).

Column w lives at b/x + w*N (internal) or b/x + w*xcol_stride (remote); b and x share both strides, as they share rhs_stride in substitute. The columns are solved in passes of pass_width columns; within a pass, every L/U tile is loaded once for all the columns of the pass. Per-column arithmetic is identical whatever the cutting, and identical to nrhs separate substitute calls: results match them bitwise (nrhs = 1 collapses to substitute exactly).

Template Parameters

nrhs	total number of right-hand-side columns
internal_rhs	residency of b and x
internal_piv	residency of piv
internal_matrix	residency of A
pass_width	columns solved together per pass; the default 0, or any value >= nrhs, means one single pass of nrhs columns

Parameters

[in]	A	factored matrix produced by factorize
[in]	A_stride	element stride of A (external mode)
[in]	piv	permutation produced by factorize
[in]	piv_stride	element stride of piv (external mode)
[in]	b	nrhs right-hand-side columns, in original (unpermuted) order
[out]	x	nrhs solution columns
[in]	rhs_stride	element stride of b and x (external mode)
[in]	xcol_stride	element stride between columns of b and x (external mode)

◆ substitute_multirhs_pass()

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int pass_width, bool internal_rhs, bool internal_piv, bool internal_matrix>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void tdls::TiledLUppSolverStatic< T, N, Config >::substitute_multirhs_pass	(	const T *TDLS_RESTRICT	A,
		const int	A_stride,
		const int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		const T *TDLS_RESTRICT	b,
		T *TDLS_RESTRICT	x,
		const int	rhs_stride,
		const int	xcol_stride )

inlinestaticconstexprnoexcept

One pass of the multi right-hand-side substitution: pass_width columns of b gathered in permuted order into x, then one triangular sweep for all pass_width columns together. Internal engine of substitute_multirhs.

Template Parameters

pass_width	number of columns of this pass
internal_rhs	residency of b and x
internal_piv	residency of piv
internal_matrix	residency of A

Parameters

[in]	A	factored matrix produced by factorize
[in]	A_stride	element stride of A (external mode)
[in]	piv	permutation produced by factorize
[in]	piv_stride	element stride of piv (external mode)
[in]	b	pass_width right-hand-side columns, in original (unpermuted) order
[out]	x	pass_width solution columns
[in]	rhs_stride	element stride of b and x (external mode)
[in]	xcol_stride	element stride between columns of b and x (external mode)

Member Data Documentation

◆ full_tiles

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

int tdls::TiledLUppSolverStatic< T, N, Config >::full_tiles = N / tile_size

staticconstexpr

full tiles per dimension

◆ schedule

template<typename T , int N, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

Schedule tdls::TiledLUppSolverStatic< T, N, Config >::schedule

staticconstexpr

Initial value:

=

Config.schedule

elimination schedule (RightLooking or LeftLooking)

The documentation for this struct was generated from the following file:

include/tdls/solvers/tiled_lupp/solver_static.hpp

Public Types

Static Public Member Functions

Static Public Attributes

Detailed Description

Member Function Documentation

◆ bwd_only()

◆ bwd_pull_one()

◆ bwd_step()

◆ factor_diag_column()

◆ factor_diag_tile()

◆ factorize() [1/2]

◆ factorize() [2/2]

◆ fwd_bwd()

◆ fwd_push_one()

◆ fwd_step()

◆ ll_correct_tile()

◆ ll_step()

◆ ll_update_below_one()

◆ ll_update_right_one()

◆ load_tile()

◆ load_tile_piv()

◆ load_tile_piv_lower()

◆ load_tile_piv_upper()

◆ rl_schur_one()

◆ rl_step()

◆ rl_trsm_right_one()

◆ rl_update_row_one()

◆ solve() [1/2]

◆ solve() [2/2]

◆ solve_inplace() [1/2]

◆ solve_inplace() [2/2]

◆ solve_inplace_multirhs() [1/2]

◆ solve_inplace_multirhs() [2/2]

◆ solve_multirhs() [1/2]

◆ solve_multirhs() [2/2]

◆ store_tile()

◆ store_tile_piv()

◆ substitute()

◆ substitute_canonical()

◆ substitute_canonical_multirhs()

◆ substitute_canonical_multirhs_pass()

◆ substitute_inplace()

◆ substitute_inplace_multirhs()

◆ substitute_inplace_multirhs_pass()

◆ substitute_multirhs()

◆ substitute_multirhs_pass()

Member Data Documentation

◆ full_tiles

◆ schedule