Runtime-size tiled dense LU factorization with logical partial pivoting and out-of-tile pivot recovery, solving one n x n system per call. More...

#include <solver_dynamic.hpp>

Static Public Member Functions
TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr int	num_tiles (const int n) noexcept
	Number of tiles per dimension (last one possibly partial).

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr int	tile_size_at (const int t0, const int n) noexcept
	Extent of the tile starting at row/column t0: tile_size, or less for the last tile.

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void	ops_swap_rows (T *TDLS_RESTRICT t, const int k, const int r, const int ke) noexcept
	Row swap k <-> r inside the tile (direct, first ke columns).

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void	ops_eliminate_column (T *TDLS_RESTRICT t, const int k, const int re, const int ce) noexcept
	Gaussian elimination of column k inside the diagonal tile.

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void	ops_trsm_left_unit (const T TDLS_RESTRICT lu, T TDLS_RESTRICT B, const int kd, const int ce) noexcept
	B := L^-1 B, with L the unit lower part of the factored diagonal tile. L is kd x kd, B is kd x ce.

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void	ops_trsm_right (const T TDLS_RESTRICT lu, T TDLS_RESTRICT B, const int kd, const int re) noexcept
	B := B U^-1, with U the upper part of the factored diagonal tile. U is kd x kd, B is re x kd.

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void	ops_gemm_sub (T TDLS_RESTRICT Ct, const T TDLS_RESTRICT At, const T *TDLS_RESTRICT Bt, const int re, const int ce, const int kd) noexcept
	Ct -= At*Bt with per-element dot-product accumulation. At is re x kd, Bt is kd x ce, Ct is re x ce.

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void	load_tile (const int n, const T TDLS_RESTRICT A, const int A_stride, const int TDLS_RESTRICT prow, const int col0, T *TDLS_RESTRICT t, const int re, const int ce) noexcept
	Load a re x ce tile through a cached physical-row segment.

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void	store_tile (const int n, T TDLS_RESTRICT A, const int A_stride, const int TDLS_RESTRICT prow, const int col0, const T *TDLS_RESTRICT t, const int re, const int ce) noexcept
	Store a re x ce tile through a cached physical-row segment.

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void	load_tile_piv (const int n, const T TDLS_RESTRICT A, const int A_stride, const int TDLS_RESTRICT piv, const int piv_stride, const int row0, const int col0, T *TDLS_RESTRICT t, const int re, const int ce) noexcept
	Load a re x ce tile, reading the permutation inline (one read per row).

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void	store_tile_piv (const int n, T TDLS_RESTRICT A, const int A_stride, const int TDLS_RESTRICT piv, const int piv_stride, const int row0, const int col0, const T *TDLS_RESTRICT t, const int re, const int ce) noexcept
	Store a re x ce tile, reading the permutation inline (one read per row).

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void	load_tile_piv_lower (const int n, const T TDLS_RESTRICT A, const int A_stride, const int TDLS_RESTRICT piv, const int piv_stride, const int row0, const int col0, T *TDLS_RESTRICT t, const int re) noexcept
	Triangular variant of the diagonal-tile load for the forward substitution: only the strict lower triangle of L\U is read.

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void	load_tile_piv_upper (const int n, const T TDLS_RESTRICT A, const int A_stride, const int TDLS_RESTRICT piv, const int piv_stride, const int row0, const int col0, T *TDLS_RESTRICT t, const int re) noexcept
	Triangular variant of the diagonal-tile load for the backward substitution: only the upper triangle including the diagonal is read.

template<bool oot_diagnostics, bool fuse_rhs>
TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr bool	factor_diag_column (const int n, T TDLS_RESTRICT A, const int A_stride, int TDLS_RESTRICT piv, const int piv_stride, const int k0, T TDLS_RESTRICT tile, int &oot_count, const int c, const int ke, T TDLS_RESTRICT y, const int rhs_stride) noexcept
	One column step of the diagonal-tile factorization: pivot search (in-tile, then out-of-tile recovery), permutation update, row swap or cross-tile pull, column elimination.

template<bool oot_diagnostics, bool fuse_rhs>
TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr bool	factor_diag_tile (const int n, T TDLS_RESTRICT A, const int A_stride, int TDLS_RESTRICT piv, const int piv_stride, const int k0, T TDLS_RESTRICT tile, int &oot_count, const int ke, T TDLS_RESTRICT y=nullptr, const int rhs_stride=1) noexcept
	Factor the ke x ke diagonal tile in registers, with out-of-tile pivot recovery (drives the per-column loop of factor_diag_column).

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void	rl_trsm_right_one (const int n, T TDLS_RESTRICT A, const int A_stride, const int TDLS_RESTRICT pk, const T *TDLS_RESTRICT tile, const int j0, const int ke, const int je) noexcept
	RL: one row-panel tile update, Akj := L^-1 Akj.

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void	rl_schur_one (const int n, T TDLS_RESTRICT A, const int A_stride, const int TDLS_RESTRICT pk, const int TDLS_RESTRICT pi, const T TDLS_RESTRICT Aik, const int j0, const int ke, const int ie, const int je) noexcept
	RL: one Schur-complement tile update, Aij -= Aik * Akj, streaming the factored Akj row by row from remote memory.

template<bool fuse_rhs>
TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void	rl_update_row_one (const int n, T TDLS_RESTRICT A, const int A_stride, const int TDLS_RESTRICT piv, const int piv_stride, const int TDLS_RESTRICT pk, const T TDLS_RESTRICT tile, const int k, const int i0, const int ke, const int ie, T *TDLS_RESTRICT y=nullptr, const int rhs_stride=1) noexcept
	RL: TRSM down + Schur sweep of one row block below the diagonal, with the optional fused forward-substitution push.

template<bool oot_diagnostics, bool fuse_rhs>
TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr bool	rl_step (const int n, T TDLS_RESTRICT A, const int A_stride, int TDLS_RESTRICT piv, const int piv_stride, const int k, int &oot_count, T *TDLS_RESTRICT y=nullptr, const int rhs_stride=1) noexcept
	RL: one full factorization step (diagonal tile + trailing updates).

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void	ll_correct_tile (const int n, const T TDLS_RESTRICT A, const int A_stride, const int TDLS_RESTRICT piv, const int piv_stride, const int row0, const int col0, const int k0, T *TDLS_RESTRICT t, const int re, const int ce) noexcept
	LL: t (re x ce, rows row0.., cols col0..) -= sum over prior tiles bj < k0/tile_size of L(row0.., bj) * U(bj, col0..). All prior tiles are full.

template<bool fuse_rhs>
TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void	ll_update_below_one (const int n, T TDLS_RESTRICT A, const int A_stride, const int TDLS_RESTRICT piv, const int piv_stride, const T TDLS_RESTRICT tile, const int k0, const int i0, const int ke, const int ie, T TDLS_RESTRICT y=nullptr, const int rhs_stride=1) noexcept
	LL: correct + TRSM one L-panel tile below the diagonal, with the optional fused forward-substitution push.

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void	ll_update_right_one (const int n, T TDLS_RESTRICT A, const int A_stride, const int TDLS_RESTRICT piv, const int piv_stride, const T *TDLS_RESTRICT tile, const int k0, const int j0, const int ke, const int je) noexcept
	LL: correct + TRSM one U-panel tile right of the diagonal.

template<bool oot_diagnostics, bool fuse_rhs>
TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr bool	ll_step (const int n, T TDLS_RESTRICT A, const int A_stride, int TDLS_RESTRICT piv, const int piv_stride, const int k, int &oot_count, T *TDLS_RESTRICT y=nullptr, const int rhs_stride=1) noexcept
	LL: one full factorization step (correct + factor the diagonal tile, then its L and U panels).

template<bool oot_diagnostics = true, bool fuse_rhs = false>
TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr bool	factorize (const int n, T TDLS_RESTRICT A, const int A_stride, int TDLS_RESTRICT piv, const int piv_stride, int &oot_count, T *TDLS_RESTRICT y=nullptr, const int rhs_stride=1) noexcept
	Factor A := PLU in place.

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr bool	factorize (const int n, T TDLS_RESTRICT A, const int A_stride, int TDLS_RESTRICT piv, const int piv_stride) noexcept
	Diagnostics-free factorize overload: no out-of-tile out-parameter at all.

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void	fwd_push_one (const int n, const int nrhs, const T TDLS_RESTRICT A, const int A_stride, const int TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const int xcol_stride, const int k0, const int m0, const int ke, const int me) noexcept
	Forward push: subtract L(m,k) * x_k from the x_m segment, for nrhs columns at once.

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void	fwd_step (const int n, const int nrhs, const T TDLS_RESTRICT A, const int A_stride, const int TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const int xcol_stride, const int k) noexcept
	Forward step: unit-lower solve the diagonal tile's segment, then push it into the tiles below.

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void	bwd_pull_one (const int n, const int nrhs, const T TDLS_RESTRICT A, const int A_stride, const int TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const int xcol_stride, const int k0, const int m0, const int ke, const int me) noexcept
	Backward pull: subtract U(k,m) * x_m from the x_k segment, for nrhs columns at once.

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void	bwd_step (const int n, const int nrhs, const T TDLS_RESTRICT A, const int A_stride, const int TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const int xcol_stride, const int k) noexcept
	Backward step: pull the trailing contributions, then upper-solve the diagonal tile's segment.

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void	bwd_only (const int n, const int nrhs, const T TDLS_RESTRICT A, const int A_stride, const int TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const int xcol_stride) noexcept
	Backward pass alone, used by fwd_bwd and by solve_inplace (whose forward pass happens inside the factorization).

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void	fwd_bwd (const int n, const int nrhs, const T TDLS_RESTRICT A, const int A_stride, const int TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const int xcol_stride) noexcept
	Triangular solves on already-permuted column(s) x. nrhs columns are processed per tile visit, so every L/U tile is loaded once for the whole block instead of once per column (nrhs = 1 = the single-RHS case).

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void	permute_inplace (const int n, const int TDLS_RESTRICT piv, const int piv_stride, T TDLS_RESTRICT x, const int rhs_stride) noexcept
	In-place application of the permutation to one strided column x, by cycle decomposition: the permute half of substitute_inplace, shared with substitute_inplace_multirhs (which applies it to each of its columns).

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void	substitute (const int n, const T TDLS_RESTRICT A, const int A_stride, const int TDLS_RESTRICT piv, const int piv_stride, const T TDLS_RESTRICT b, T TDLS_RESTRICT x, const int rhs_stride) noexcept
	Solve x := U^-1 L^-1 P b from a prior factorize. b and x must not alias (use substitute_inplace for the aliased single-buffer pattern).

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void	substitute_canonical (const int n, const T TDLS_RESTRICT A, const int A_stride, const int TDLS_RESTRICT piv, const int piv_stride, const int col, T *TDLS_RESTRICT x, const int rhs_stride) noexcept
	Solve with b = e_col generated on the fly: the consistent-tangent-operator path.

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void	substitute_canonical_multirhs_pass (const int n, const int nrhs, const T TDLS_RESTRICT A, const int A_stride, const int TDLS_RESTRICT piv, const int piv_stride, const int col0, T *TDLS_RESTRICT x, const int rhs_stride, const int xcol_stride) noexcept
	One pass of the canonical multi right-hand-side substitution: nrhs canonical columns e_col0 .. e_{col0+nrhs-1} generated in permuted order into x, then one triangular sweep for all nrhs columns together. Internal engine of substitute_canonical_multirhs.

template<int pass_width = 0>
TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void	substitute_canonical_multirhs (const int n, const int nrhs, const T TDLS_RESTRICT A, const int A_stride, const int TDLS_RESTRICT piv, const int piv_stride, const int col0, T *TDLS_RESTRICT x, const int rhs_stride, const int xcol_stride) noexcept
	nrhs canonical columns e_col0 .. e_{col0+nrhs-1} solved from a prior factorize: the columns of the inverse that a consistent tangent operator needs.

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void	substitute_multirhs_pass (const int n, const int nrhs, const T TDLS_RESTRICT A, const int A_stride, const int TDLS_RESTRICT piv, const int piv_stride, const T TDLS_RESTRICT b, T TDLS_RESTRICT x, const int rhs_stride, const int xcol_stride) noexcept
	One pass of the multi right-hand-side substitution: nrhs columns of b gathered in permuted order into x, then one triangular sweep for all nrhs columns together. Internal engine of substitute_multirhs.

template<int pass_width = 0>
TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void	substitute_multirhs (const int n, const int nrhs, const T TDLS_RESTRICT A, const int A_stride, const int TDLS_RESTRICT piv, const int piv_stride, const T TDLS_RESTRICT b, T TDLS_RESTRICT x, const int rhs_stride, const int xcol_stride) noexcept
	Solve nrhs right-hand-side columns from a prior factorize: X := U^-1 L^-1 P B, column by column. b and x must not alias (use substitute_inplace_multirhs for the aliased single-buffer pattern).

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void	substitute_inplace (const int n, const T TDLS_RESTRICT A, const int A_stride, const int TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride) noexcept
	Solve in place: x already holds the unpermuted RHS on entry and the solution on exit.

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void	substitute_inplace_multirhs_pass (const int n, const int nrhs, const T TDLS_RESTRICT A, const int A_stride, const int TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const int xcol_stride) noexcept
	One pass of the in-place multi right-hand-side substitution: nrhs columns of x permuted by the cycle decomposition of permute_inplace, then one triangular sweep for all nrhs columns together. Internal engine of substitute_inplace_multirhs.

template<int pass_width = 0>
TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void	substitute_inplace_multirhs (const int n, const int nrhs, const T TDLS_RESTRICT A, const int A_stride, const int TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT x, const int rhs_stride, const int xcol_stride) noexcept
	nrhs columns solved in place: x holds the nrhs unpermuted right-hand-side columns on entry and the nrhs solutions on exit.

template<bool oot_diagnostics = true>
TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr bool	solve (const int n, T TDLS_RESTRICT A, const int A_stride, int TDLS_RESTRICT piv, const int piv_stride, const T TDLS_RESTRICT b, T TDLS_RESTRICT x, const int rhs_stride, int &oot_count) noexcept
	factorize + substitute in one call.

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr bool	solve (const int n, T TDLS_RESTRICT A, const int A_stride, int TDLS_RESTRICT piv, const int piv_stride, const T TDLS_RESTRICT b, T TDLS_RESTRICT x, const int rhs_stride) noexcept
	Diagnostics-free solve overload: no out-of-tile out-parameter at all.

template<int pass_width = 0, bool oot_diagnostics = true>
TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr bool	solve_multirhs (const int n, const int nrhs, T TDLS_RESTRICT A, const int A_stride, int TDLS_RESTRICT piv, const int piv_stride, const T TDLS_RESTRICT b, T TDLS_RESTRICT x, const int rhs_stride, const int xcol_stride, int &oot_count) noexcept
	factorize + substitute_multirhs in one call.

template<int pass_width = 0>
TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr bool	solve_multirhs (const int n, const int nrhs, T TDLS_RESTRICT A, const int A_stride, int TDLS_RESTRICT piv, const int piv_stride, const T TDLS_RESTRICT b, T TDLS_RESTRICT x, const int rhs_stride, const int xcol_stride) noexcept
	Diagnostics-free solve_multirhs overload: no out-of-tile out-parameter at all.

template<bool oot_diagnostics = true>
TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr bool	solve_inplace (const int n, T TDLS_RESTRICT A, const int A_stride, int TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT y, const int rhs_stride, int &oot_count) noexcept
	Factorization with the forward substitution folded in.

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr bool	solve_inplace (const int n, T TDLS_RESTRICT A, const int A_stride, int TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT y, const int rhs_stride) noexcept
	Diagnostics-free solve_inplace overload: no out-of-tile out-parameter at all.

template<int pass_width = 0, bool oot_diagnostics = true>
TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr bool	solve_inplace_multirhs (const int n, const int nrhs, T TDLS_RESTRICT A, const int A_stride, int TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT y, const int rhs_stride, const int xcol_stride, int &oot_count) noexcept
	factorize + substitute_inplace_multirhs in one call: y holds the nrhs unpermuted right-hand-side columns on entry and the nrhs solutions on exit.

template<int pass_width = 0>
TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr bool	solve_inplace_multirhs (const int n, const int nrhs, T TDLS_RESTRICT A, const int A_stride, int TDLS_RESTRICT piv, const int piv_stride, T *TDLS_RESTRICT y, const int rhs_stride, const int xcol_stride) noexcept
	Diagnostics-free solve_inplace_multirhs overload: no out-of-tile out-parameter at all.

Static Public Attributes
static constexpr int	tile_size = Config.tile_size
	tile size (int)

static constexpr Schedule	schedule
	elimination schedule (RightLooking or LeftLooking)

static constexpr T	oot_threshold = Config.oot_threshold
	Acceptable-pivot threshold of the out-of-tile search, read once from the configuration (see TiledLUppConfig::oot_threshold).

static constexpr T	singular_floor = Config.singular_floor
	Singularity floor of the out-of-tile recovery, read once from the configuration (see TiledLUppConfig::singular_floor).

Detailed Description

template<typename T, TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>
struct tdls::TiledLUppSolverDynamic< T, Config >

Runtime-size tiled dense LU factorization with logical partial pivoting and out-of-tile pivot recovery, solving one n x n system per call.

All entry points are static, host- and device-callable, and take the dimension n plus raw pointers pre-offset by the caller with one runtime stride per array. Entry points:

factorize: A := P*L*U in place, pivots out (RL or LL)
substitute: x := U^-1 L^-1 P b (b and x distinct)
substitute_canonical: idem with b = e_col (tangent-operator columns)
substitute_inplace: idem with b == x (cycle-decomposition permute)
solve: factorize + substitute
solve_inplace: factorize with the forward pass folded in

Each substitution and solve entry point has a _multirhs twin taking nrhs right-hand-side columns per call (substitute_multirhs, substitute_canonical_multirhs, substitute_inplace_multirhs, solve_multirhs, solve_inplace_multirhs). The columns are solved in passes of pass_width columns (0 = one single pass); every L/U tile is loaded once per pass, and per-column results match nrhs single-column calls bitwise whatever the cutting. nrhs is a runtime argument here, as every dimension of this variant (the compile-time solver takes it as a template parameter, under the same name). pass_width is a scheduling knob, not a dimension: it stays a template parameter defaulting to 0, as in the compile-time solver and in the adaptors, so the common case needs no cutting argument at all.

The factored diagonal holds the RECIPROCALS of the U pivots, exactly as in TiledLUppSolverStatic: a factorization produced here must be consumed by the substitution routines of this family.

Template Parameters

T	scalar type (float, double or long double)
Config	compile-time knobs, passed as a constexpr value: tile size (may exceed n), schedule, pivoting thresholds; see TiledLUppConfig (unroll_inner is ignored by this variant)

Member Function Documentation

◆ bwd_only()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void tdls::TiledLUppSolverDynamic< T, Config >::bwd_only	(	const int	n,
		const int	nrhs,
		const T *TDLS_RESTRICT	A,
		const int	A_stride,
		const int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		T *TDLS_RESTRICT	x,
		const int	rhs_stride,
		const int	xcol_stride )

inlinestaticconstexprnoexcept

Backward pass alone, used by fwd_bwd and by solve_inplace (whose forward pass happens inside the factorization).

Parameters

[in]	n	system dimension
[in]	nrhs	number of columns processed together
[in]	A	factored matrix (caller-pre-offset)
[in]	A_stride	element stride of A
[in]	piv	permutation from factorize
[in]	piv_stride	element stride of piv
[in,out]	x	solution column(s)
[in]	rhs_stride	element stride of x
[in]	xcol_stride	element stride between columns of x

◆ bwd_pull_one()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void tdls::TiledLUppSolverDynamic< T, Config >::bwd_pull_one	(	const int	n,
		const int	nrhs,
		const T *TDLS_RESTRICT	A,
		const int	A_stride,
		const int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		T *TDLS_RESTRICT	x,
		const int	rhs_stride,
		const int	xcol_stride,
		const int	k0,
		const int	m0,
		const int	ke,
		const int	me )

inlinestaticconstexprnoexcept

Backward pull: subtract U(k,m) * x_m from the x_k segment, for nrhs columns at once.

Parameters

[in]	n	system dimension
[in]	nrhs	number of columns processed together
[in]	A	factored matrix (caller-pre-offset)
[in]	A_stride	element stride of A
[in]	piv	permutation from factorize
[in]	piv_stride	element stride of piv
[in,out]	x	solution column(s)
[in]	rhs_stride	element stride of x
[in]	xcol_stride	element stride between columns of x
[in]	k0	first global row of the updated segment
[in]	m0	first global row of the trailing segment
[in]	ke	extent of the updated segment's tile
[in]	me	extent of the trailing segment's tile

◆ bwd_step()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void tdls::TiledLUppSolverDynamic< T, Config >::bwd_step	(	const int	n,
		const int	nrhs,
		const T *TDLS_RESTRICT	A,
		const int	A_stride,
		const int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		T *TDLS_RESTRICT	x,
		const int	rhs_stride,
		const int	xcol_stride,
		const int	k )

inlinestaticconstexprnoexcept

Backward step: pull the trailing contributions, then upper-solve the diagonal tile's segment.

Parameters

[in]	n	system dimension
[in]	nrhs	number of columns processed together
[in]	A	factored matrix (caller-pre-offset)
[in]	A_stride	element stride of A
[in]	piv	permutation from factorize
[in]	piv_stride	element stride of piv
[in,out]	x	solution column(s)
[in]	rhs_stride	element stride of x
[in]	xcol_stride	element stride between columns of x
[in]	k	step index (k0 = k*tile_size)

◆ factor_diag_column()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<bool oot_diagnostics, bool fuse_rhs>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr bool tdls::TiledLUppSolverDynamic< T, Config >::factor_diag_column	(	const int	n,
		T *TDLS_RESTRICT	A,
		const int	A_stride,
		int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		const int	k0,
		T *TDLS_RESTRICT	tile,
		int &	oot_count,
		const int	c,
		const int	ke,
		T *TDLS_RESTRICT	y,
		const int	rhs_stride )

inlinestaticnodiscardconstexprnoexcept

One column step of the diagonal-tile factorization: pivot search (in-tile, then out-of-tile recovery), permutation update, row swap or cross-tile pull, column elimination.

Template Parameters

oot_diagnostics	compile the out-of-tile counter in or out
fuse_rhs	apply the pivot swaps to the fused RHS y

Parameters

[in]	n	system dimension
[in,out]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A
[in,out]	piv	permutation (logical -> physical row)
[in]	piv_stride	element stride of piv
[in]	k0	first global row/column of the tile
[in,out]	tile	register-resident diagonal tile
[in,out]	oot_count	out-of-tile search counter (oot_diagnostics)
[in]	c	tile column to factor
[in]	ke	extent of the diagonal tile
[in,out]	y	fused right-hand side (fuse_rhs only)
[in]	rhs_stride	element stride of y

Returns: false when the matrix is singular at this column.

◆ factor_diag_tile()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<bool oot_diagnostics, bool fuse_rhs>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr bool tdls::TiledLUppSolverDynamic< T, Config >::factor_diag_tile	(	const int	n,
		T *TDLS_RESTRICT	A,
		const int	A_stride,
		int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		const int	k0,
		T *TDLS_RESTRICT	tile,
		int &	oot_count,
		const int	ke,
		T *TDLS_RESTRICT	y = nullptr,
		const int	rhs_stride = 1 )

inlinestaticnodiscardconstexprnoexcept

Factor the ke x ke diagonal tile in registers, with out-of-tile pivot recovery (drives the per-column loop of factor_diag_column).

Template Parameters

oot_diagnostics	compile the out-of-tile counter in or out
fuse_rhs	apply the pivot swaps to the fused RHS y

Parameters

[in]	n	system dimension
[in,out]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A
[in,out]	piv	permutation (logical -> physical row)
[in]	piv_stride	element stride of piv
[in]	k0	first global row/column of the tile
[in,out]	tile	register-resident diagonal tile, loaded (and, LL: prior-corrected) by the caller
[in,out]	oot_count	out-of-tile search counter (oot_diagnostics)
[in]	ke	extent of the diagonal tile
[in,out]	y	fused right-hand side (fuse_rhs only)
[in]	rhs_stride	element stride of y

Returns: false on a singular matrix.

◆ factorize() [1/2]

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr bool tdls::TiledLUppSolverDynamic< T, Config >::factorize	(	const int	n,
		T *TDLS_RESTRICT	A,
		const int	A_stride,
		int *TDLS_RESTRICT	piv,
		const int	piv_stride )

inlinestaticnodiscardconstexprnoexcept

Diagnostics-free factorize overload: no out-of-tile out-parameter at all.

Parameters

[in]	n	system dimension (n >= 1)
[in,out]	A	matrix, pre-offset by the caller
[in]	A_stride	element stride of A
[out]	piv	permutation storage (always caller-provided)
[in]	piv_stride	element stride of piv

Returns: false on a singular matrix.

◆ factorize() [2/2]

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<bool oot_diagnostics = true, bool fuse_rhs = false>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr bool tdls::TiledLUppSolverDynamic< T, Config >::factorize	(	const int	n,
		T *TDLS_RESTRICT	A,
		const int	A_stride,
		int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		int &	oot_count,
		T *TDLS_RESTRICT	y = nullptr,
		const int	rhs_stride = 1 )

inlinestaticnodiscardconstexprnoexcept

Factor A := P*L*U in place.

A becomes L\U (unit lower L below the diagonal, U above including it, the diagonal holding the pivot RECIPROCALS), under logical row permutation piv (piv[i] = physical row holding logical row i).

Template Parameters

oot_diagnostics	when false, the out-of-tile diagnostics are compiled out entirely; use the overload without the out-parameter
fuse_rhs	internal hook of solve_inplace: folds the forward substitution of y into the factorization

Parameters

[in]	n	system dimension (n >= 1)
[in,out]	A	matrix, pre-offset by the caller
[in]	A_stride	element stride of A
[out]	piv	permutation storage (always caller-provided)
[in]	piv_stride	element stride of piv
[out]	oot_count	number of columns whose best in-tile pivot fell below oot_threshold
[in,out]	y	fused right-hand side (fuse_rhs only)
[in]	rhs_stride	element stride of y

Returns: false on a singular matrix.

◆ fwd_bwd()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void tdls::TiledLUppSolverDynamic< T, Config >::fwd_bwd	(	const int	n,
		const int	nrhs,
		const T *TDLS_RESTRICT	A,
		const int	A_stride,
		const int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		T *TDLS_RESTRICT	x,
		const int	rhs_stride,
		const int	xcol_stride )

inlinestaticconstexprnoexcept

Triangular solves on already-permuted column(s) x. nrhs columns are processed per tile visit, so every L/U tile is loaded once for the whole block instead of once per column (nrhs = 1 = the single-RHS case).

Parameters

[in]	n	system dimension
[in]	nrhs	number of columns processed together
[in]	A	factored matrix (caller-pre-offset)
[in]	A_stride	element stride of A
[in]	piv	permutation from factorize
[in]	piv_stride	element stride of piv
[in,out]	x	solution column(s)
[in]	rhs_stride	element stride of x
[in]	xcol_stride	element stride between columns of x

◆ fwd_push_one()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void tdls::TiledLUppSolverDynamic< T, Config >::fwd_push_one	(	const int	n,
		const int	nrhs,
		const T *TDLS_RESTRICT	A,
		const int	A_stride,
		const int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		T *TDLS_RESTRICT	x,
		const int	rhs_stride,
		const int	xcol_stride,
		const int	k0,
		const int	m0,
		const int	ke,
		const int	me )

inlinestaticconstexprnoexcept

Forward push: subtract L(m,k) * x_k from the x_m segment, for nrhs columns at once.

Parameters

[in]	n	system dimension
[in]	nrhs	number of columns processed together
[in]	A	factored matrix (caller-pre-offset)
[in]	A_stride	element stride of A
[in]	piv	permutation from factorize
[in]	piv_stride	element stride of piv
[in,out]	x	solution column(s)
[in]	rhs_stride	element stride of x
[in]	xcol_stride	element stride between columns of x
[in]	k0	first global row of the solved segment
[in]	m0	first global row of the target segment
[in]	ke	extent of the solved segment's tile
[in]	me	extent of the target segment's tile

◆ fwd_step()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void tdls::TiledLUppSolverDynamic< T, Config >::fwd_step	(	const int	n,
		const int	nrhs,
		const T *TDLS_RESTRICT	A,
		const int	A_stride,
		const int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		T *TDLS_RESTRICT	x,
		const int	rhs_stride,
		const int	xcol_stride,
		const int	k )

inlinestaticconstexprnoexcept

Forward step: unit-lower solve the diagonal tile's segment, then push it into the tiles below.

Parameters

[in]	n	system dimension
[in]	nrhs	number of columns processed together
[in]	A	factored matrix (caller-pre-offset)
[in]	A_stride	element stride of A
[in]	piv	permutation from factorize
[in]	piv_stride	element stride of piv
[in,out]	x	solution column(s)
[in]	rhs_stride	element stride of x
[in]	xcol_stride	element stride between columns of x
[in]	k	step index (k0 = k*tile_size)

◆ ll_correct_tile()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void tdls::TiledLUppSolverDynamic< T, Config >::ll_correct_tile	(	const int	n,
		const T *TDLS_RESTRICT	A,
		const int	A_stride,
		const int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		const int	row0,
		const int	col0,
		const int	k0,
		T *TDLS_RESTRICT	t,
		const int	re,
		const int	ce )

inlinestaticconstexprnoexcept

LL: t (re x ce, rows row0.., cols col0..) -= sum over prior tiles bj < k0/tile_size of L(row0.., bj) * U(bj, col0..). All prior tiles are full.

Parameters

[in]	n	system dimension
[in]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A
[in]	piv	permutation (logical -> physical row)
[in]	piv_stride	element stride of piv
[in]	row0	first global row of the corrected tile
[in]	col0	first global column of the corrected tile
[in]	k0	first row/column of the current step
[in,out]	t	register tile being corrected
[in]	re	row extent of the corrected tile
[in]	ce	column extent of the corrected tile

◆ ll_step()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<bool oot_diagnostics, bool fuse_rhs>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr bool tdls::TiledLUppSolverDynamic< T, Config >::ll_step	(	const int	n,
		T *TDLS_RESTRICT	A,
		const int	A_stride,
		int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		const int	k,
		int &	oot_count,
		T *TDLS_RESTRICT	y = nullptr,
		const int	rhs_stride = 1 )

inlinestaticnodiscardconstexprnoexcept

LL: one full factorization step (correct + factor the diagonal tile, then its L and U panels).

Template Parameters

oot_diagnostics	compile the out-of-tile counter in or out
fuse_rhs	apply the step to the fused RHS y (solve_inplace)

Parameters

[in]	n	system dimension
[in,out]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A
[in,out]	piv	permutation (logical -> physical row)
[in]	piv_stride	element stride of piv
[in]	k	step index (k0 = k*tile_size)
[in,out]	oot_count	out-of-tile search counter (oot_diagnostics)
[in,out]	y	fused right-hand side (fuse_rhs only)
[in]	rhs_stride	element stride of y

Returns: false on a singular matrix.

◆ ll_update_below_one()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<bool fuse_rhs>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void tdls::TiledLUppSolverDynamic< T, Config >::ll_update_below_one	(	const int	n,
		T *TDLS_RESTRICT	A,
		const int	A_stride,
		const int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		const T *TDLS_RESTRICT	tile,
		const int	k0,
		const int	i0,
		const int	ke,
		const int	ie,
		T *TDLS_RESTRICT	y = nullptr,
		const int	rhs_stride = 1 )

inlinestaticconstexprnoexcept

LL: correct + TRSM one L-panel tile below the diagonal, with the optional fused forward-substitution push.

Template Parameters

fuse_rhs push the fused RHS y along (solve_inplace)

Parameters

[in]	n	system dimension
[in,out]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A
[in]	piv	permutation (logical -> physical row)
[in]	piv_stride	element stride of piv
[in]	tile	factored diagonal tile
[in]	k0	first global row/column of the step
[in]	i0	first global row of the updated tile
[in]	ke	extent of the diagonal tile
[in]	ie	row extent of the updated tile
[in,out]	y	fused right-hand side (fuse_rhs only)
[in]	rhs_stride	element stride of y

◆ ll_update_right_one()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void tdls::TiledLUppSolverDynamic< T, Config >::ll_update_right_one	(	const int	n,
		T *TDLS_RESTRICT	A,
		const int	A_stride,
		const int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		const T *TDLS_RESTRICT	tile,
		const int	k0,
		const int	j0,
		const int	ke,
		const int	je )

inlinestaticconstexprnoexcept

LL: correct + TRSM one U-panel tile right of the diagonal.

Parameters

[in]	n	system dimension
[in,out]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A
[in]	piv	permutation (logical -> physical row)
[in]	piv_stride	element stride of piv
[in]	tile	factored diagonal tile
[in]	k0	first global row/column of the step
[in]	j0	first global column of the updated tile
[in]	ke	extent of the diagonal tile
[in]	je	column extent of the updated tile

◆ load_tile()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void tdls::TiledLUppSolverDynamic< T, Config >::load_tile	(	const int	n,
		const T *TDLS_RESTRICT	A,
		const int	A_stride,
		const int *TDLS_RESTRICT	prow,
		const int	col0,
		T *TDLS_RESTRICT	t,
		const int	re,
		const int	ce )

inlinestaticconstexprnoexcept

Load a re x ce tile through a cached physical-row segment.

Parameters

[in]	n	system dimension
[in]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A
[in]	prow	physical rows of the tile
[in]	col0	first global column of the tile
[out]	t	destination register tile
[in]	re	tile row extent
[in]	ce	tile column extent

◆ load_tile_piv()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void tdls::TiledLUppSolverDynamic< T, Config >::load_tile_piv	(	const int	n,
		const T *TDLS_RESTRICT	A,
		const int	A_stride,
		const int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		const int	row0,
		const int	col0,
		T *TDLS_RESTRICT	t,
		const int	re,
		const int	ce )

inlinestaticconstexprnoexcept

Load a re x ce tile, reading the permutation inline (one read per row).

Parameters

[in]	n	system dimension
[in]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A
[in]	piv	permutation (logical -> physical row)
[in]	piv_stride	element stride of piv
[in]	row0	first global (logical) row of the tile
[in]	col0	first global column of the tile
[out]	t	destination register tile
[in]	re	tile row extent
[in]	ce	tile column extent

◆ load_tile_piv_lower()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void tdls::TiledLUppSolverDynamic< T, Config >::load_tile_piv_lower	(	const int	n,
		const T *TDLS_RESTRICT	A,
		const int	A_stride,
		const int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		const int	row0,
		const int	col0,
		T *TDLS_RESTRICT	t,
		const int	re )

inlinestaticconstexprnoexcept

Triangular variant of the diagonal-tile load for the forward substitution: only the strict lower triangle of L\U is read.

Parameters

[in]	n	system dimension
[in]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A
[in]	piv	permutation (logical -> physical row)
[in]	piv_stride	element stride of piv
[in]	row0	first global (logical) row of the tile
[in]	col0	first global column of the tile
[out]	t	destination register tile
[in]	re	tile extent

◆ load_tile_piv_upper()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void tdls::TiledLUppSolverDynamic< T, Config >::load_tile_piv_upper	(	const int	n,
		const T *TDLS_RESTRICT	A,
		const int	A_stride,
		const int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		const int	row0,
		const int	col0,
		T *TDLS_RESTRICT	t,
		const int	re )

inlinestaticconstexprnoexcept

Triangular variant of the diagonal-tile load for the backward substitution: only the upper triangle including the diagonal is read.

Parameters

[in]	n	system dimension
[in]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A
[in]	piv	permutation (logical -> physical row)
[in]	piv_stride	element stride of piv
[in]	row0	first global (logical) row of the tile
[in]	col0	first global column of the tile
[out]	t	destination register tile
[in]	re	tile extent

◆ num_tiles()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr int tdls::TiledLUppSolverDynamic< T, Config >::num_tiles ( const int n )

inlinestaticconstexprnoexcept

Number of tiles per dimension (last one possibly partial).

Parameters

[in] n system dimension

◆ ops_eliminate_column()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void tdls::TiledLUppSolverDynamic< T, Config >::ops_eliminate_column	(	T *TDLS_RESTRICT	t,
		const int	k,
		const int	re,
		const int	ce )

inlinestaticconstexprnoexcept

Gaussian elimination of column k inside the diagonal tile.

Scales the sub-column by 1/pivot and updates the trailing block. The RECIPROCAL of the pivot is stored at the diagonal slot: every downstream consumer multiplies instead of dividing.

Parameters

[in,out]	t	register tile
[in]	k	column to eliminate
[in]	re	active row extent of the tile
[in]	ce	active column extent of the tile

◆ ops_gemm_sub()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void tdls::TiledLUppSolverDynamic< T, Config >::ops_gemm_sub	(	T *TDLS_RESTRICT	Ct,
		const T *TDLS_RESTRICT	At,
		const T *TDLS_RESTRICT	Bt,
		const int	re,
		const int	ce,
		const int	kd )

inlinestaticconstexprnoexcept

Ct -= At*Bt with per-element dot-product accumulation. At is re x kd, Bt is kd x ce, Ct is re x ce.

Parameters

[in,out]	Ct	accumulator tile
[in]	At	left factor tile
[in]	Bt	right factor tile
[in]	re	row extent of Ct and At
[in]	ce	column extent of Ct and Bt
[in]	kd	inner extent (columns of At, rows of Bt)

◆ ops_swap_rows()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void tdls::TiledLUppSolverDynamic< T, Config >::ops_swap_rows	(	T *TDLS_RESTRICT	t,
		const int	k,
		const int	r,
		const int	ke )

inlinestaticconstexprnoexcept

Row swap k <-> r inside the tile (direct, first ke columns).

Parameters

[in,out]	t	register tile
[in]	k	destination row (elimination column)
[in]	r	source row to swap in
[in]	ke	active extent of the tile

◆ ops_trsm_left_unit()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void tdls::TiledLUppSolverDynamic< T, Config >::ops_trsm_left_unit	(	const T *TDLS_RESTRICT	lu,
		T *TDLS_RESTRICT	B,
		const int	kd,
		const int	ce )

inlinestaticconstexprnoexcept

B := L^-1 B, with L the unit lower part of the factored diagonal tile. L is kd x kd, B is kd x ce.

Parameters

[in]	lu	factored diagonal tile (L\U)
[in,out]	B	updated register tile
[in]	kd	extent of the factored diagonal tile
[in]	ce	column extent of B

◆ ops_trsm_right()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void tdls::TiledLUppSolverDynamic< T, Config >::ops_trsm_right	(	const T *TDLS_RESTRICT	lu,
		T *TDLS_RESTRICT	B,
		const int	kd,
		const int	re )

inlinestaticconstexprnoexcept

B := B U^-1, with U the upper part of the factored diagonal tile. U is kd x kd, B is re x kd.

The diagonal of lu already holds the pivot reciprocals: no divisions here.

Parameters

[in]	lu	factored diagonal tile (L\U, reciprocal diagonal)
[in,out]	B	updated register tile
[in]	kd	extent of the factored diagonal tile
[in]	re	row extent of B

◆ permute_inplace()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void tdls::TiledLUppSolverDynamic< T, Config >::permute_inplace	(	const int	n,
		const int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		T *TDLS_RESTRICT	x,
		const int	rhs_stride )

inlinestaticconstexprnoexcept

In-place application of the permutation to one strided column x, by cycle decomposition: the permute half of substitute_inplace, shared with substitute_inplace_multirhs (which applies it to each of its columns).

Each cycle is rotated once from its smallest index. Up to n = 64 a 64-bit bitmask tracks the visited rows; beyond, a cycle-leader scan is used instead (zero storage, an extra integer walk of each orbit). The dispatch is warp-uniform on GPU (n is the same for every lane). Both paths move the same values in the same order.

Parameters

[in]	n	system dimension
[in]	piv	permutation from factorize
[in]	piv_stride	element stride of piv
[in,out]	x	column to permute
[in]	rhs_stride	element stride of x

◆ rl_schur_one()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void tdls::TiledLUppSolverDynamic< T, Config >::rl_schur_one	(	const int	n,
		T *TDLS_RESTRICT	A,
		const int	A_stride,
		const int *TDLS_RESTRICT	pk,
		const int *TDLS_RESTRICT	pi,
		const T *TDLS_RESTRICT	Aik,
		const int	j0,
		const int	ke,
		const int	ie,
		const int	je )

inlinestaticconstexprnoexcept

RL: one Schur-complement tile update, Aij -= Aik * Akj, streaming the factored Akj row by row from remote memory.

Parameters

[in]	n	system dimension
[in,out]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A
[in]	pk	physical rows of the factored block row
[in]	pi	physical rows of the updated block row
[in]	Aik	register L panel of the updated block row
[in]	j0	first global column of the updated tile
[in]	ke	inner extent (current step)
[in]	ie	row extent of the updated tile
[in]	je	column extent of the updated tile

◆ rl_step()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<bool oot_diagnostics, bool fuse_rhs>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr bool tdls::TiledLUppSolverDynamic< T, Config >::rl_step	(	const int	n,
		T *TDLS_RESTRICT	A,
		const int	A_stride,
		int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		const int	k,
		int &	oot_count,
		T *TDLS_RESTRICT	y = nullptr,
		const int	rhs_stride = 1 )

inlinestaticnodiscardconstexprnoexcept

RL: one full factorization step (diagonal tile + trailing updates).

Template Parameters

oot_diagnostics	compile the out-of-tile counter in or out
fuse_rhs	apply the step to the fused RHS y (solve_inplace)

Parameters

[in]	n	system dimension
[in,out]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A
[in,out]	piv	permutation (logical -> physical row)
[in]	piv_stride	element stride of piv
[in]	k	step index (k0 = k*tile_size)
[in,out]	oot_count	out-of-tile search counter (oot_diagnostics)
[in,out]	y	fused right-hand side (fuse_rhs only)
[in]	rhs_stride	element stride of y

Returns: false on a singular matrix.

◆ rl_trsm_right_one()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void tdls::TiledLUppSolverDynamic< T, Config >::rl_trsm_right_one	(	const int	n,
		T *TDLS_RESTRICT	A,
		const int	A_stride,
		const int *TDLS_RESTRICT	pk,
		const T *TDLS_RESTRICT	tile,
		const int	j0,
		const int	ke,
		const int	je )

inlinestaticconstexprnoexcept

RL: one row-panel tile update, Akj := L^-1 Akj.

Parameters

[in]	n	system dimension
[in,out]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A
[in]	pk	physical rows of the diagonal block row
[in]	tile	factored diagonal tile
[in]	j0	first global column of the updated tile
[in]	ke	extent of the factored diagonal tile
[in]	je	column extent of the updated tile

◆ rl_update_row_one()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<bool fuse_rhs>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void tdls::TiledLUppSolverDynamic< T, Config >::rl_update_row_one	(	const int	n,
		T *TDLS_RESTRICT	A,
		const int	A_stride,
		const int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		const int *TDLS_RESTRICT	pk,
		const T *TDLS_RESTRICT	tile,
		const int	k,
		const int	i0,
		const int	ke,
		const int	ie,
		T *TDLS_RESTRICT	y = nullptr,
		const int	rhs_stride = 1 )

inlinestaticconstexprnoexcept

RL: TRSM down + Schur sweep of one row block below the diagonal, with the optional fused forward-substitution push.

Template Parameters

fuse_rhs push the fused RHS y along (solve_inplace)

Parameters

[in]	n	system dimension
[in,out]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A
[in]	piv	permutation (logical -> physical row)
[in]	piv_stride	element stride of piv
[in]	pk	physical rows of the diagonal tile
[in]	tile	factored diagonal tile
[in]	k	step index (k0 = k*tile_size)
[in]	i0	first global row of the updated block row
[in]	ke	extent of the diagonal tile
[in]	ie	row extent of the updated block row
[in,out]	y	fused right-hand side (fuse_rhs only)
[in]	rhs_stride	element stride of y

◆ solve() [1/2]

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr bool tdls::TiledLUppSolverDynamic< T, Config >::solve	(	const int	n,
		T *TDLS_RESTRICT	A,
		const int	A_stride,
		int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		const T *TDLS_RESTRICT	b,
		T *TDLS_RESTRICT	x,
		const int	rhs_stride )

inlinestaticnodiscardconstexprnoexcept

Diagnostics-free solve overload: no out-of-tile out-parameter at all.

Parameters

[in]	n	system dimension (n >= 1)
[in,out]	A	on entry the matrix (pre-offset by the caller), on exit its factorization
[in]	A_stride	element stride of A
[out]	piv	permutation storage (always caller-provided)
[in]	piv_stride	element stride of piv
[in]	b	right-hand side, in original order
[out]	x	solution
[in]	rhs_stride	element stride of b and x

Returns: false on a singular matrix.

◆ solve() [2/2]

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<bool oot_diagnostics = true>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr bool tdls::TiledLUppSolverDynamic< T, Config >::solve	(	const int	n,
		T *TDLS_RESTRICT	A,
		const int	A_stride,
		int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		const T *TDLS_RESTRICT	b,
		T *TDLS_RESTRICT	x,
		const int	rhs_stride,
		int &	oot_count )

inlinestaticnodiscardconstexprnoexcept

factorize + substitute in one call.

Template Parameters

oot_diagnostics compile the out-of-tile counter in or out

Parameters

[in]	n	system dimension (n >= 1)
[in,out]	A	on entry the matrix (pre-offset by the caller), on exit its factorization (usable for further substitute* calls)
[in]	A_stride	element stride of A
[out]	piv	permutation storage (always caller-provided)
[in]	piv_stride	element stride of piv
[in]	b	right-hand side, in original order
[out]	x	solution
[in]	rhs_stride	element stride of b and x
[out]	oot_count	number of columns whose best in-tile pivot fell below oot_threshold

Returns: false on a singular matrix.

◆ solve_inplace() [1/2]

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr bool tdls::TiledLUppSolverDynamic< T, Config >::solve_inplace	(	const int	n,
		T *TDLS_RESTRICT	A,
		const int	A_stride,
		int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		T *TDLS_RESTRICT	y,
		const int	rhs_stride )

inlinestaticnodiscardconstexprnoexcept

Diagnostics-free solve_inplace overload: no out-of-tile out-parameter at all.

Parameters

[in]	n	system dimension (n >= 1)
[in,out]	A	on entry the matrix (pre-offset by the caller), on exit its factorization
[in]	A_stride	element stride of A
[out]	piv	permutation storage (always caller-provided)
[in]	piv_stride	element stride of piv
[in,out]	y	right-hand side on entry, solution on exit
[in]	rhs_stride	element stride of y

Returns: false on a singular matrix (y left partially updated).

◆ solve_inplace() [2/2]

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<bool oot_diagnostics = true>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr bool tdls::TiledLUppSolverDynamic< T, Config >::solve_inplace	(	const int	n,
		T *TDLS_RESTRICT	A,
		const int	A_stride,
		int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		T *TDLS_RESTRICT	y,
		const int	rhs_stride,
		int &	oot_count )

inlinestaticnodiscardconstexprnoexcept

Factorization with the forward substitution folded in.

y holds the unpermuted RHS on entry and the solution on exit. Each L panel is applied to y while it sits in registers during the factorization, so the separate forward pass disappears, and all its tile reloads with it; only the backward pass remains. y follows the rows through pivoting (swap hook in factor_diag_column), which makes the result bitwise-identical to factorize + substitute_inplace.

Template Parameters

oot_diagnostics compile the out-of-tile counter in or out

Parameters

[in]	n	system dimension (n >= 1)
[in,out]	A	on entry the matrix (pre-offset by the caller), on exit its factorization
[in]	A_stride	element stride of A
[out]	piv	permutation storage (always caller-provided)
[in]	piv_stride	element stride of piv
[in,out]	y	right-hand side on entry, solution on exit
[in]	rhs_stride	element stride of y
[out]	oot_count	number of columns whose best in-tile pivot fell below oot_threshold

Returns: false on a singular matrix (y left partially updated).

◆ solve_inplace_multirhs() [1/2]

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int pass_width = 0>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr bool tdls::TiledLUppSolverDynamic< T, Config >::solve_inplace_multirhs	(	const int	n,
		const int	nrhs,
		T *TDLS_RESTRICT	A,
		const int	A_stride,
		int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		T *TDLS_RESTRICT	y,
		const int	rhs_stride,
		const int	xcol_stride )

inlinestaticnodiscardconstexprnoexcept

Diagnostics-free solve_inplace_multirhs overload: no out-of-tile out-parameter at all.

Template Parameters

pass_width columns solved together per substitution pass; the default 0, or any value >= nrhs, means one single pass

Parameters

[in]	n	system dimension (n >= 1)
[in]	nrhs	total number of right-hand-side columns
[in,out]	A	on entry the matrix (pre-offset by the caller), on exit its factorization
[in]	A_stride	element stride of A
[out]	piv	permutation storage (always caller-provided)
[in]	piv_stride	element stride of piv
[in,out]	y	nrhs right-hand-side columns on entry, nrhs solution columns on exit
[in]	rhs_stride	element stride of y
[in]	xcol_stride	element stride between columns of y

Returns: false on a singular matrix (y left untouched).

◆ solve_inplace_multirhs() [2/2]

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int pass_width = 0, bool oot_diagnostics = true>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr bool tdls::TiledLUppSolverDynamic< T, Config >::solve_inplace_multirhs	(	const int	n,
		const int	nrhs,
		T *TDLS_RESTRICT	A,
		const int	A_stride,
		int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		T *TDLS_RESTRICT	y,
		const int	rhs_stride,
		const int	xcol_stride,
		int &	oot_count )

inlinestaticnodiscardconstexprnoexcept

factorize + substitute_inplace_multirhs in one call: y holds the nrhs unpermuted right-hand-side columns on entry and the nrhs solutions on exit.

Unlike solve_inplace, the forward pass is not folded into the factorization: a pass amortizes the forward tile loads across its columns, which removes most of what the folding saves.

Template Parameters

pass_width	columns solved together per substitution pass; the default 0, or any value >= nrhs, means one single pass
oot_diagnostics	compile the out-of-tile counter in or out

Parameters

[in]	n	system dimension (n >= 1)
[in]	nrhs	total number of right-hand-side columns
[in,out]	A	on entry the matrix (pre-offset by the caller), on exit its factorization
[in]	A_stride	element stride of A
[out]	piv	permutation storage (always caller-provided)
[in]	piv_stride	element stride of piv
[in,out]	y	nrhs right-hand-side columns on entry, nrhs solution columns on exit
[in]	rhs_stride	element stride of y
[in]	xcol_stride	element stride between columns of y
[out]	oot_count	number of columns whose best in-tile pivot fell below oot_threshold

Returns: false on a singular matrix (y left untouched).

◆ solve_multirhs() [1/2]

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int pass_width = 0>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr bool tdls::TiledLUppSolverDynamic< T, Config >::solve_multirhs	(	const int	n,
		const int	nrhs,
		T *TDLS_RESTRICT	A,
		const int	A_stride,
		int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		const T *TDLS_RESTRICT	b,
		T *TDLS_RESTRICT	x,
		const int	rhs_stride,
		const int	xcol_stride )

inlinestaticnodiscardconstexprnoexcept

Diagnostics-free solve_multirhs overload: no out-of-tile out-parameter at all.

Template Parameters

pass_width columns solved together per substitution pass; the default 0, or any value >= nrhs, means one single pass

Parameters

[in]	n	system dimension (n >= 1)
[in]	nrhs	total number of right-hand-side columns
[in,out]	A	on entry the matrix (pre-offset by the caller), on exit its factorization
[in]	A_stride	element stride of A
[out]	piv	permutation storage (always caller-provided)
[in]	piv_stride	element stride of piv
[in]	b	nrhs right-hand-side columns, in original order
[out]	x	nrhs solution columns
[in]	rhs_stride	element stride of b and x
[in]	xcol_stride	element stride between columns of b and x

Returns: false on a singular matrix.

◆ solve_multirhs() [2/2]

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int pass_width = 0, bool oot_diagnostics = true>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr bool tdls::TiledLUppSolverDynamic< T, Config >::solve_multirhs	(	const int	n,
		const int	nrhs,
		T *TDLS_RESTRICT	A,
		const int	A_stride,
		int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		const T *TDLS_RESTRICT	b,
		T *TDLS_RESTRICT	x,
		const int	rhs_stride,
		const int	xcol_stride,
		int &	oot_count )

inlinestaticnodiscardconstexprnoexcept

factorize + substitute_multirhs in one call.

Template Parameters

pass_width	columns solved together per substitution pass; the default 0, or any value >= nrhs, means one single pass
oot_diagnostics	compile the out-of-tile counter in or out

Parameters

[in]	n	system dimension (n >= 1)
[in]	nrhs	total number of right-hand-side columns
[in,out]	A	on entry the matrix (pre-offset by the caller), on exit its factorization (usable for further substitute* calls)
[in]	A_stride	element stride of A
[out]	piv	permutation storage (always caller-provided)
[in]	piv_stride	element stride of piv
[in]	b	nrhs right-hand-side columns, in original order
[out]	x	nrhs solution columns
[in]	rhs_stride	element stride of b and x
[in]	xcol_stride	element stride between columns of b and x
[out]	oot_count	number of columns whose best in-tile pivot fell below oot_threshold

Returns: false on a singular matrix.

◆ store_tile()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void tdls::TiledLUppSolverDynamic< T, Config >::store_tile	(	const int	n,
		T *TDLS_RESTRICT	A,
		const int	A_stride,
		const int *TDLS_RESTRICT	prow,
		const int	col0,
		const T *TDLS_RESTRICT	t,
		const int	re,
		const int	ce )

inlinestaticconstexprnoexcept

Store a re x ce tile through a cached physical-row segment.

Parameters

[in]	n	system dimension
[in,out]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A
[in]	prow	physical rows of the tile
[in]	col0	first global column of the tile
[in]	t	source register tile
[in]	re	tile row extent
[in]	ce	tile column extent

◆ store_tile_piv()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void tdls::TiledLUppSolverDynamic< T, Config >::store_tile_piv	(	const int	n,
		T *TDLS_RESTRICT	A,
		const int	A_stride,
		const int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		const int	row0,
		const int	col0,
		const T *TDLS_RESTRICT	t,
		const int	re,
		const int	ce )

inlinestaticconstexprnoexcept

Store a re x ce tile, reading the permutation inline (one read per row).

Parameters

[in]	n	system dimension
[in,out]	A	matrix (caller-pre-offset)
[in]	A_stride	element stride of A
[in]	piv	permutation (logical -> physical row)
[in]	piv_stride	element stride of piv
[in]	row0	first global (logical) row of the tile
[in]	col0	first global column of the tile
[in]	t	source register tile
[in]	re	tile row extent
[in]	ce	tile column extent

◆ substitute()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void tdls::TiledLUppSolverDynamic< T, Config >::substitute	(	const int	n,
		const T *TDLS_RESTRICT	A,
		const int	A_stride,
		const int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		const T *TDLS_RESTRICT	b,
		T *TDLS_RESTRICT	x,
		const int	rhs_stride )

inlinestaticconstexprnoexcept

Solve x := U^-1 L^-1 P b from a prior factorize. b and x must not alias (use substitute_inplace for the aliased single-buffer pattern).

Parameters

[in]	n	system dimension
[in]	A	factored matrix produced by factorize
[in]	A_stride	element stride of A
[in]	piv	permutation produced by factorize
[in]	piv_stride	element stride of piv
[in]	b	right-hand side, in original (unpermuted) order
[out]	x	solution
[in]	rhs_stride	element stride of b and x

◆ substitute_canonical()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void tdls::TiledLUppSolverDynamic< T, Config >::substitute_canonical	(	const int	n,
		const T *TDLS_RESTRICT	A,
		const int	A_stride,
		const int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		const int	col,
		T *TDLS_RESTRICT	x,
		const int	rhs_stride )

inlinestaticconstexprnoexcept

Solve with b = e_col generated on the fly: the consistent-tangent-operator path.

Thin alias of the canonical pass engine below with nrhs = 1, which collapses to exactly the single-column code, so there is no separate implementation to maintain.

Parameters

[in]	n	system dimension
[in]	A	factored matrix produced by factorize
[in]	A_stride	element stride of A
[in]	piv	permutation produced by factorize
[in]	piv_stride	element stride of piv
[in]	col	index of the canonical column e_col
[out]	x	solution
[in]	rhs_stride	element stride of x

◆ substitute_canonical_multirhs()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int pass_width = 0>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void tdls::TiledLUppSolverDynamic< T, Config >::substitute_canonical_multirhs	(	const int	n,
		const int	nrhs,
		const T *TDLS_RESTRICT	A,
		const int	A_stride,
		const int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		const int	col0,
		T *TDLS_RESTRICT	x,
		const int	rhs_stride,
		const int	xcol_stride )

inlinestaticconstexprnoexcept

nrhs canonical columns e_col0 .. e_{col0+nrhs-1} solved from a prior factorize: the columns of the inverse that a consistent tangent operator needs.

Column w lives at x + w*xcol_stride. The columns are solved in passes of pass_width columns; within a pass, every L/U tile is loaded once for all the columns of the pass. Per-column arithmetic is identical whatever the cutting, and identical to nrhs separate substitute_canonical calls: results match them bitwise (nrhs = 1 collapses to substitute_canonical exactly).

Template Parameters

pass_width columns solved together per pass; the default 0, or any value >= nrhs, means one single pass of nrhs columns

Parameters

[in]	n	system dimension
[in]	nrhs	number of consecutive canonical columns
[in]	A	factored matrix produced by factorize
[in]	A_stride	element stride of A
[in]	piv	permutation produced by factorize
[in]	piv_stride	element stride of piv
[in]	col0	index of the first canonical column
[out]	x	nrhs solution columns
[in]	rhs_stride	element stride of x
[in]	xcol_stride	element stride between columns of x

◆ substitute_canonical_multirhs_pass()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void tdls::TiledLUppSolverDynamic< T, Config >::substitute_canonical_multirhs_pass	(	const int	n,
		const int	nrhs,
		const T *TDLS_RESTRICT	A,
		const int	A_stride,
		const int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		const int	col0,
		T *TDLS_RESTRICT	x,
		const int	rhs_stride,
		const int	xcol_stride )

inlinestaticconstexprnoexcept

One pass of the canonical multi right-hand-side substitution: nrhs canonical columns e_col0 .. e_{col0+nrhs-1} generated in permuted order into x, then one triangular sweep for all nrhs columns together. Internal engine of substitute_canonical_multirhs.

Parameters

[in]	n	system dimension
[in]	nrhs	number of columns of this pass
[in]	A	factored matrix produced by factorize
[in]	A_stride	element stride of A
[in]	piv	permutation produced by factorize
[in]	piv_stride	element stride of piv
[in]	col0	index of the first canonical column
[out]	x	nrhs solution columns
[in]	rhs_stride	element stride of x
[in]	xcol_stride	element stride between columns of x

◆ substitute_inplace()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void tdls::TiledLUppSolverDynamic< T, Config >::substitute_inplace	(	const int	n,
		const T *TDLS_RESTRICT	A,
		const int	A_stride,
		const int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		T *TDLS_RESTRICT	x,
		const int	rhs_stride )

inlinestaticconstexprnoexcept

Solve in place: x already holds the unpermuted RHS on entry and the solution on exit.

The permutation is applied in place by cycle decomposition (see permute_inplace).

Parameters

[in]	n	system dimension
[in]	A	factored matrix produced by factorize
[in]	A_stride	element stride of A
[in]	piv	permutation produced by factorize
[in]	piv_stride	element stride of piv
[in,out]	x	right-hand side on entry, solution on exit
[in]	rhs_stride	element stride of x

◆ substitute_inplace_multirhs()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int pass_width = 0>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void tdls::TiledLUppSolverDynamic< T, Config >::substitute_inplace_multirhs	(	const int	n,
		const int	nrhs,
		const T *TDLS_RESTRICT	A,
		const int	A_stride,
		const int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		T *TDLS_RESTRICT	x,
		const int	rhs_stride,
		const int	xcol_stride )

inlinestaticconstexprnoexcept

nrhs columns solved in place: x holds the nrhs unpermuted right-hand-side columns on entry and the nrhs solutions on exit.

The permutation is applied to each column by the cycle decomposition of permute_inplace. The columns are solved in passes of pass_width columns; within a pass, every L/U tile is loaded once for all the columns of the pass. Per-column values are identical whatever the cutting, and match nrhs separate substitute_inplace calls bitwise (nrhs = 1 collapses to substitute_inplace exactly).

Template Parameters

pass_width columns solved together per pass; the default 0, or any value >= nrhs, means one single pass of nrhs columns

Parameters

[in]	n	system dimension
[in]	nrhs	total number of right-hand-side columns
[in]	A	factored matrix produced by factorize
[in]	A_stride	element stride of A
[in]	piv	permutation produced by factorize
[in]	piv_stride	element stride of piv
[in,out]	x	nrhs right-hand-side columns on entry, nrhs solution columns on exit
[in]	rhs_stride	element stride of x
[in]	xcol_stride	element stride between columns of x

◆ substitute_inplace_multirhs_pass()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void tdls::TiledLUppSolverDynamic< T, Config >::substitute_inplace_multirhs_pass	(	const int	n,
		const int	nrhs,
		const T *TDLS_RESTRICT	A,
		const int	A_stride,
		const int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		T *TDLS_RESTRICT	x,
		const int	rhs_stride,
		const int	xcol_stride )

inlinestaticconstexprnoexcept

One pass of the in-place multi right-hand-side substitution: nrhs columns of x permuted by the cycle decomposition of permute_inplace, then one triangular sweep for all nrhs columns together. Internal engine of substitute_inplace_multirhs.

Parameters

[in]	n	system dimension
[in]	nrhs	number of columns of this pass
[in]	A	factored matrix produced by factorize
[in]	A_stride	element stride of A
[in]	piv	permutation produced by factorize
[in]	piv_stride	element stride of piv
[in,out]	x	nrhs right-hand-side columns on entry, nrhs solution columns on exit
[in]	rhs_stride	element stride of x
[in]	xcol_stride	element stride between columns of x

◆ substitute_multirhs()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

template<int pass_width = 0>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void tdls::TiledLUppSolverDynamic< T, Config >::substitute_multirhs	(	const int	n,
		const int	nrhs,
		const T *TDLS_RESTRICT	A,
		const int	A_stride,
		const int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		const T *TDLS_RESTRICT	b,
		T *TDLS_RESTRICT	x,
		const int	rhs_stride,
		const int	xcol_stride )

inlinestaticconstexprnoexcept

Solve nrhs right-hand-side columns from a prior factorize: X := U^-1 L^-1 P B, column by column. b and x must not alias (use substitute_inplace_multirhs for the aliased single-buffer pattern).

Column w lives at b/x + w*xcol_stride; b and x share both strides, as they share rhs_stride in substitute. The columns are solved in passes of pass_width columns; within a pass, every L/U tile is loaded once for all the columns of the pass. Per-column arithmetic is identical whatever the cutting, and identical to nrhs separate substitute calls: results match them bitwise (nrhs = 1 collapses to substitute exactly).

Template Parameters

pass_width columns solved together per pass; the default 0, or any value >= nrhs, means one single pass of nrhs columns

Parameters

[in]	n	system dimension
[in]	nrhs	total number of right-hand-side columns
[in]	A	factored matrix produced by factorize
[in]	A_stride	element stride of A
[in]	piv	permutation produced by factorize
[in]	piv_stride	element stride of piv
[in]	b	nrhs right-hand-side columns, in original (unpermuted) order
[out]	x	nrhs solution columns
[in]	rhs_stride	element stride of b and x
[in]	xcol_stride	element stride between columns of b and x

◆ substitute_multirhs_pass()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr void tdls::TiledLUppSolverDynamic< T, Config >::substitute_multirhs_pass	(	const int	n,
		const int	nrhs,
		const T *TDLS_RESTRICT	A,
		const int	A_stride,
		const int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		const T *TDLS_RESTRICT	b,
		T *TDLS_RESTRICT	x,
		const int	rhs_stride,
		const int	xcol_stride )

inlinestaticconstexprnoexcept

One pass of the multi right-hand-side substitution: nrhs columns of b gathered in permuted order into x, then one triangular sweep for all nrhs columns together. Internal engine of substitute_multirhs.

Parameters

[in]	n	system dimension
[in]	nrhs	number of columns of this pass
[in]	A	factored matrix produced by factorize
[in]	A_stride	element stride of A
[in]	piv	permutation produced by factorize
[in]	piv_stride	element stride of piv
[in]	b	nrhs right-hand-side columns, in original (unpermuted) order
[out]	x	nrhs solution columns
[in]	rhs_stride	element stride of b and x
[in]	xcol_stride	element stride between columns of b and x

◆ tile_size_at()

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCEINLINE constexpr int tdls::TiledLUppSolverDynamic< T, Config >::tile_size_at	(	const int	t0,
		const int	n )

inlinestaticconstexprnoexcept

Extent of the tile starting at row/column t0: tile_size, or less for the last tile.

Parameters

[in]	t0	first global row/column of the tile
[in]	n	system dimension

Member Data Documentation

◆ schedule

template<typename T , TiledLUppConfig< T > Config = TiledLUppConfig<T>{}>

Schedule tdls::TiledLUppSolverDynamic< T, Config >::schedule

staticconstexpr

Initial value:

=

Config.schedule

elimination schedule (RightLooking or LeftLooking)

The documentation for this struct was generated from the following file:

include/tdls/solvers/tiled_lupp/solver_dynamic.hpp

Static Public Member Functions

Static Public Attributes

Detailed Description

Member Function Documentation

◆ bwd_only()

◆ bwd_pull_one()

◆ bwd_step()

◆ factor_diag_column()

◆ factor_diag_tile()

◆ factorize() [1/2]

◆ factorize() [2/2]

◆ fwd_bwd()

◆ fwd_push_one()

◆ fwd_step()

◆ ll_correct_tile()

◆ ll_step()

◆ ll_update_below_one()

◆ ll_update_right_one()

◆ load_tile()

◆ load_tile_piv()

◆ load_tile_piv_lower()

◆ load_tile_piv_upper()

◆ num_tiles()

◆ ops_eliminate_column()

◆ ops_gemm_sub()

◆ ops_swap_rows()

◆ ops_trsm_left_unit()

◆ ops_trsm_right()

◆ permute_inplace()

◆ rl_schur_one()

◆ rl_step()

◆ rl_trsm_right_one()

◆ rl_update_row_one()

◆ solve() [1/2]

◆ solve() [2/2]

◆ solve_inplace() [1/2]

◆ solve_inplace() [2/2]

◆ solve_inplace_multirhs() [1/2]

◆ solve_inplace_multirhs() [2/2]

◆ solve_multirhs() [1/2]

◆ solve_multirhs() [2/2]

◆ store_tile()

◆ store_tile_piv()

◆ substitute()

◆ substitute_canonical()

◆ substitute_canonical_multirhs()

◆ substitute_canonical_multirhs_pass()

◆ substitute_inplace()

◆ substitute_inplace_multirhs()

◆ substitute_inplace_multirhs_pass()

◆ substitute_multirhs()

◆ substitute_multirhs_pass()

◆ tile_size_at()

Member Data Documentation

◆ schedule