Dense LU factorization with partial pivoting held in the registers of a group of threads, solving one NxN system per group. More...

#include <solver_static.hpp>

Static Public Member Functions
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool	slot_has_rows (const int K) noexcept
	Whether slot K holds a real row for at least one thread.

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool	slot_is_full (const int K) noexcept
	Whether slot K holds a real row for every thread.

template<typename Fn >
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	at_position (const int tx, const int pos, Fn &&fn) noexcept
	Calls fn on the slot of static index pos when the calling thread holds it: the position of a row under physical row interchanges, the entry of the right-hand side under the transposed solve.

template<bool internal_matrix>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	load_rows (const int tx, const T *TDLS_RESTRICT A, const int A_stride, T(&rA)[rows_per_thread][N]) noexcept
	Load the rows of the calling thread into registers.

template<bool internal_matrix>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	store_rows (const int tx, T *TDLS_RESTRICT A, const int A_stride, const T(&rA)[rows_per_thread][N]) noexcept
	Store the register rows of the calling thread.

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	init_piv (const int tx, int(&rpiv)[rows_per_thread]) noexcept
	Initial pivot entries: no row has moved yet, so the entry of each slot is the index of its row, phantom rows included (theirs stays at or above N). Under logical row interchanges the entry is the position of the row of the slot, under physical ones the original index of the row held by the slot: both start there.

template<bool internal_piv>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	load_piv (const int tx, const int *TDLS_RESTRICT piv, const int piv_stride, int(&rpiv)[rows_per_thread]) noexcept
	Load the pivot entries recorded by factorize.

template<bool internal_piv>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	store_piv (const int tx, int *TDLS_RESTRICT piv, const int piv_stride, const int(&rpiv)[rows_per_thread]) noexcept
	Record the pivot entries of the calling thread.

template<bool internal_rhs>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	load_rhs (const int tx, const T *TDLS_RESTRICT b, const int rhs_stride, T(&rB)[rows_per_thread]) noexcept
	Load the right-hand-side entries of the rows of the calling thread.

template<bool internal_rhs>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	store_solution (const int tx, T *TDLS_RESTRICT x, const int rhs_stride, const T(&rB)[rows_per_thread]) noexcept
	Store the solution entries held by the calling thread.

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool	keeps_pivot (const T in_place, const T column_max) noexcept
	Relative threshold of the pivot choice: whether the row in place keeps the pivot (see CooperativeLUppConfig::relative_pivot_threshold).

template<bool fuse_rhs, typename Sync >
TDLS_EXEC_CHECK_DISABLE TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	logical_elimination_step (const int tx, const int i, T(&rA)[rows_per_thread][N], T(&rB)[rows_per_thread], int(&rowid)[rows_per_thread], T sB, T sx, T *dsx, int &linfo, Sync &sync) noexcept(detail::nothrow_sync< Sync >)
	One column of the factorization under logical row interchanges.

template<bool fuse_rhs>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	save_row (const T(&row)[N], const T &rhs, const int origin, T *record) noexcept
	Copy a register row to an exchange record of the workspace: its N entries, its right-hand-side entry (fuse_rhs only), then its original index. The index travels as a value of T, exact since the offset bounds keep N far below the mantissa range of every scalar type.

template<bool fuse_rhs>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	load_row (T(&row)[N], T &rhs, int &origin, const T *record) noexcept
	Copy an exchange record of the workspace to a register row, the converse of save_row.

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool	below (const int tx, const int K, const int i) noexcept
	Whether slot K of the calling thread holds a real row that may sit below column i: the slots a pivot row below the diagonal can come from, a compile-time choice in an unrolled column sweep but for the rank test of a mixed slot.

template<bool fuse_rhs>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	gather_row (const int tx, const int i, const int pos, const T(&rA)[rows_per_thread][N], const T(&rB)[rows_per_thread], const int(&origin)[rows_per_thread], T *record) noexcept
	Copy the register row at position pos below column i, held by the calling thread in a slot known at run time only, to an exchange record, as save_row does.

template<bool fuse_rhs>
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	scatter_row (const int tx, const int i, const int pos, T(&rA)[rows_per_thread][N], T(&rB)[rows_per_thread], int(&origin)[rows_per_thread], const T *record) noexcept
	Copy an exchange record to the register row at position pos below column i, held by the calling thread in a slot known at run time only, the converse of gather_row: each slot takes the record or keeps its row through a select.

template<bool fuse_rhs, typename Sync >
TDLS_EXEC_CHECK_DISABLE TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	physical_elimination_step (const int tx, const int i, T(&rA)[rows_per_thread][N], T(&rB)[rows_per_thread], int(&origin)[rows_per_thread], T *work, int &linfo, Sync &sync) noexcept(detail::nothrow_sync< Sync >)
	One column of the factorization under physical row interchanges.

template<bool fuse_rhs, typename Sync >
TDLS_EXEC_CHECK_DISABLE TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr int	eliminate (const int tx, T(&rA)[rows_per_thread][N], T(&rB)[rows_per_thread], int(&rpiv)[rows_per_thread], T *work, Sync &sync) noexcept(detail::nothrow_sync< Sync >)
	Factorization of the register rows of the group.

template<typename Sync >
TDLS_EXEC_CHECK_DISABLE TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	forward_step (const int i, const T(&rA)[rows_per_thread][N], T(&rB)[rows_per_thread], const int(&rowid)[rows_per_thread], T *sB, Sync &sync) noexcept(detail::nothrow_sync< Sync >)
	One step of the forward substitution.

template<typename Sync >
TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	forward_pass (const T(&rA)[rows_per_thread][N], T(&rB)[rows_per_thread], const int(&rowid)[rows_per_thread], T *work, Sync &sync) noexcept(detail::nothrow_sync< Sync >)
	Forward pass: L y = P b on the given row positions, y overwriting the right-hand-side registers.

template<typename Sync >
TDLS_EXEC_CHECK_DISABLE TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	backward_step (const int i, const int tx, const T(&rA)[rows_per_thread][N], T(&rB)[rows_per_thread], const int(&rowid)[rows_per_thread], T sB, T sx, Sync &sync) noexcept(detail::nothrow_sync< Sync >)
	One step of the backward substitution.

template<typename Sync >
TDLS_EXEC_CHECK_DISABLE TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	backward_pass (const int tx, const T(&rA)[rows_per_thread][N], T(&rB)[rows_per_thread], const int(&rowid)[rows_per_thread], T *work, Sync &sync) noexcept(detail::nothrow_sync< Sync >)
	Backward pass: U x = y on the given row positions, the forward-reduced right-hand side y in the registers on entry, the solution entries of the thread on exit.

template<typename Sync >
TDLS_EXEC_CHECK_DISABLE TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	gather_rhs (const int tx, T(&rB)[rows_per_thread], const int(&origin)[rows_per_thread], T *staged, Sync &sync) noexcept(detail::nothrow_sync< Sync >)
	Gather of the right-hand side in the pivoted order, under physical row interchanges: each slot receives the entry of the original row it holds.

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr T *	segment (T *work, const int step) noexcept
	Half of the double buffer of the transposed substitutions used by a step: N + 1 elements, a row segment and one value. A thread reads the half of step s after the barrier of step s; the next write to that half comes at step s + 2, after the barrier of step s + 1, which every thread reaches only once its reads of step s are done. Both halves take 2 * N + 2 elements, within the workspace for N >= 2; a dimension of 1 uses the first half only.

template<typename Sync >
TDLS_EXEC_CHECK_DISABLE TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	transposed_forward_step (const int tx, const int i, const T(&rA)[rows_per_thread][N], T(&rB)[rows_per_thread], const int(&rowid)[rows_per_thread], T *work, Sync &sync) noexcept(detail::nothrow_sync< Sync >)
	One step of the forward substitution U^T z = b.

template<typename Sync >
TDLS_EXEC_CHECK_DISABLE TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	transposed_backward_step (const int tx, const int k, const T(&rA)[rows_per_thread][N], T(&rB)[rows_per_thread], const int(&rowid)[rows_per_thread], T *work, Sync &sync) noexcept(detail::nothrow_sync< Sync >)
	One step of the backward substitution L^T w = z.

template<bool backward_only, typename Sync >
TDLS_EXEC_CHECK_DISABLE TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	transposed_passes (const int tx, const T(&rA)[rows_per_thread][N], T(&rB)[rows_per_thread], const int(&rowid)[rows_per_thread], T *work, Sync &sync) noexcept(detail::nothrow_sync< Sync >)
	Forward then backward pass of S^T x = b on the given row positions: b in the entries of the thread on entry, w at the entries of the same positions on exit. Position 0 updates no entry in the backward pass: its step is skipped.

template<typename Sync >
TDLS_EXEC_CHECK_DISABLE TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	unpermute_solution (const int tx, T(&rB)[rows_per_thread], const int(&rpiv)[rows_per_thread], T *work, Sync &sync) noexcept(detail::nothrow_sync< Sync >)
	x = P^T w, through the workspace. Logical row interchanges: the slot of row r of S takes w at the position of the row. Physical ones: w_p goes to the original index of the row at position p. The exchange is staged in the half of the double buffer that the last backward step does not read.

template<bool backward_only, typename Sync >
TDLS_EXEC_CHECK_DISABLE TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	substitution (const int tx, const T(&rA)[rows_per_thread][N], T(&rB)[rows_per_thread], const int(&rpiv)[rows_per_thread], T *work, Sync &sync) noexcept(detail::nothrow_sync< Sync >)
	Substitution from the factored rows, the right-hand side in the registers on entry, the solution entries of the thread on exit: L y = P b then U x = y under the column-major layout, U^T z = b, L^T w = z and x = P^T w under the row-major one.

TDLS_EXEC_CHECK_DISABLE TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr auto	make_sync (T *work) noexcept
	The barrier deduced for the group of the calling thread.

template<bool internal_piv, bool internal_matrix, int A_static_stride = dynamic_stride, int piv_static_stride = dynamic_stride, typename Sync = tdls::AutoSync>
TDLS_EXEC_CHECK_DISABLE TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool	factorize (const int tx, T TDLS_RESTRICT A, const int A_stride, int TDLS_RESTRICT piv, const int piv_stride, T *work, Sync &&sync=Sync{}) noexcept(detail::nothrow_sync< Sync >)
	In-place LU factorization with partial pivoting.

template<bool internal_rhs, bool internal_piv, bool internal_matrix, int A_static_stride = dynamic_stride, int piv_static_stride = dynamic_stride, int rhs_static_stride = dynamic_stride, typename Sync = tdls::AutoSync>
TDLS_EXEC_CHECK_DISABLE TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	substitute (const int tx, const T TDLS_RESTRICT A, const int A_stride, const int TDLS_RESTRICT piv, const int piv_stride, const T TDLS_RESTRICT b, T TDLS_RESTRICT x, const int rhs_stride, T *work, Sync &&sync=Sync{}) noexcept(detail::nothrow_sync< Sync >)
	Solve from a factorization produced by factorize (b and x distinct).

template<bool internal_rhs, bool internal_piv, bool internal_matrix, int A_static_stride = dynamic_stride, int piv_static_stride = dynamic_stride, int rhs_static_stride = dynamic_stride, typename Sync = tdls::AutoSync>
TDLS_EXEC_CHECK_DISABLE TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	substitute_canonical (const int tx, const T TDLS_RESTRICT A, const int A_stride, const int TDLS_RESTRICT piv, const int piv_stride, const int col, T TDLS_RESTRICT x, const int rhs_stride, T work, Sync &&sync=Sync{}) noexcept(detail::nothrow_sync< Sync >)
	Solve with b = e_col generated on the fly: the consistent-tangent-operator path.

template<bool internal_rhs, bool internal_piv, bool internal_matrix, int A_static_stride = dynamic_stride, int piv_static_stride = dynamic_stride, int rhs_static_stride = dynamic_stride, typename Sync = tdls::AutoSync>
TDLS_EXEC_CHECK_DISABLE TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void	substitute_inplace (const int tx, const T TDLS_RESTRICT A, const int A_stride, const int TDLS_RESTRICT piv, const int piv_stride, T TDLS_RESTRICT x, const int rhs_stride, T work, Sync &&sync=Sync{}) noexcept(detail::nothrow_sync< Sync >)
	Solve in place from a factorization produced by factorize: x holds the right-hand side on entry and the solution on exit.

template<bool internal_rhs, bool internal_piv, bool internal_matrix, int A_static_stride = dynamic_stride, int piv_static_stride = dynamic_stride, int rhs_static_stride = dynamic_stride, typename Sync = tdls::AutoSync>
TDLS_EXEC_CHECK_DISABLE TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool	solve (const int tx, T TDLS_RESTRICT A, const int A_stride, int TDLS_RESTRICT piv, const int piv_stride, const T TDLS_RESTRICT b, T TDLS_RESTRICT x, const int rhs_stride, T *work, Sync &&sync=Sync{}) noexcept(detail::nothrow_sync< Sync >)
	factorize + substitute in one call. The rows stay in registers between the two, so the factors are stored once and never reloaded.

template<bool internal_rhs, bool internal_piv, bool internal_matrix, int A_static_stride = dynamic_stride, int piv_static_stride = dynamic_stride, int rhs_static_stride = dynamic_stride, typename Sync = tdls::AutoSync>
TDLS_EXEC_CHECK_DISABLE TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool	solve_inplace (const int tx, T TDLS_RESTRICT A, const int A_stride, int TDLS_RESTRICT piv, const int piv_stride, T TDLS_RESTRICT y, const int rhs_stride, T work, Sync &&sync=Sync{}) noexcept(detail::nothrow_sync< Sync >)
	Factorization with the forward substitution folded in: the original MAGMA kernel.

Static Public Attributes
static constexpr int	rows_per_thread
	Rows held by each thread: Config.rows_per_thread, or N when it is larger. A value below 1, rejected above, is clamped to 1 so that the rejection stays the only diagnostic.

static constexpr int	threads_per_system = (N + rows_per_thread - 1) / rows_per_thread
	Threads solving one system: ceil(N / rows_per_thread).

static constexpr int	workspace_size
	Elements of the workspace shared by the threads of a group: the two elements of the deduced CPU barrier, then 3 * N under logical row interchanges, 2 * N + 2 * threads_per_system + 5 under physical ones.

static constexpr T	relative_pivot_threshold = Config.relative_pivot_threshold
	Relative threshold of the pivot choice, read once from the configuration (see CooperativeLUppConfig::relative_pivot_threshold).

static constexpr T	singular_floor = Config.singular_floor
	Singularity floor, read once from the configuration (see CooperativeLUppConfig::singular_floor).

static constexpr bool	transposed = Config.layout == MatrixLayout::RowMajor
	Whether the solve uses the transpose of the factored matrix. The solver factors S, the stored matrix read column-major, so that consecutive threads hold consecutive addresses whatever the layout. Under the column-major layout S = A and the solve is S x = b, the kernel of MAGMA. Under the row-major layout S = A^T, the threads hold columns of A, and the solve is S^T x = b (see the file documentation).

Detailed Description

template<typename T, int N, CooperativeLUppConfig< T > Config = CooperativeLUppConfig<T>{}>
struct tdls::CooperativeLUppSolverStatic< T, N, Config >

Dense LU factorization with partial pivoting held in the registers of a group of threads, solving one NxN system per group.

All entry points are static, host- and device-callable, called by every thread of the group with its rank tx, and take raw pointers pre-offset by the caller plus one runtime stride per array (see the file documentation for the calling convention). Entry points:

factorize: S := P*L*U in place, pivot entries out (S = A column-major, A^T row-major)
substitute: x := A^-1 b from a factorization (b and x distinct)
substitute_canonical: idem with b = e_col (tangent-operator columns)
substitute_inplace: idem with b == x
solve: factorize + substitute, the rows staying in registers in between
solve_inplace: factorize with the forward pass folded in, the original MAGMA kernel

factorize/substitute stay separate so a factorization can be reused across several right-hand sides. solve, solve_inplace and factorize followed by substitute produce bitwise-identical results.

The residency template booleans (internal_rhs / internal_piv / internal_matrix) deliberately have no default: they describe what the passed buffers ARE, so each call site must state them explicitly.

Template Parameters

T	scalar type (float, double or long double)
N	system dimension (N >= 1)
Config	compile-time knobs, passed as a constexpr value: rows per thread, row interchanges, relative pivot threshold, singularity floor, unroll policy, matrix layout; see CooperativeLUppConfig

Member Function Documentation

◆ at_position()

template<typename T , int N, CooperativeLUppConfig< T > Config = CooperativeLUppConfig<T>{}>

template<typename Fn >

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::CooperativeLUppSolverStatic< T, N, Config >::at_position	(	const int	tx,
		const int	pos,
		Fn &&	fn )

inlinestaticconstexprnoexcept

Calls fn on the slot of static index pos when the calling thread holds it: the position of a row under physical row interchanges, the entry of the right-hand side under the transposed solve.

Index pos lives in slot pos / threads_per_system of thread pos % threads_per_system. Inside an unrolled column sweep, a column index is a constant: the slot is then selected at compile time, and only the rank of the thread is tested at run time. The solver passes it only callables that never call the barrier, hence noexcept.

Template Parameters

Fn	callable type

Parameters

[in]	tx	rank of the calling thread in its group
[in]	pos	position, in [0, N)
[in]	fn	callable, invoked as fn(std::integral_constant<int, K>{})

◆ backward_pass()

template<typename T , int N, CooperativeLUppConfig< T > Config = CooperativeLUppConfig<T>{}>

template<typename Sync >

TDLS_EXEC_CHECK_DISABLE TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::CooperativeLUppSolverStatic< T, N, Config >::backward_pass	(	const int	tx,
		const T(&)	rA[rows_per_thread][N],
		T(&)	rB[rows_per_thread],
		const int(&)	rowid[rows_per_thread],
		T *	work,
		Sync &	sync )

inlinestaticconstexprnoexcept

Backward pass: U x = y on the given row positions, the forward-reduced right-hand side y in the registers on entry, the solution entries of the thread on exit.

Template Parameters

Sync	callable type of the barrier

Parameters

[in]	tx	rank of the calling thread in its group
[in]	rA	factored register rows of the thread
[in,out]	rB	y entries of the rows of the thread on entry, solution entries tx + K * threads_per_system on exit
[in]	rowid	position of each row of the thread in the pivoted order
[in]	work	workspace of the group
[in]	sync	barrier of the group

◆ backward_step()

template<typename T , int N, CooperativeLUppConfig< T > Config = CooperativeLUppConfig<T>{}>

template<typename Sync >

TDLS_EXEC_CHECK_DISABLE TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::CooperativeLUppSolverStatic< T, N, Config >::backward_step	(	const int	i,
		const int	tx,
		const T(&)	rA[rows_per_thread][N],
		T(&)	rB[rows_per_thread],
		const int(&)	rowid[rows_per_thread],
		T *	sB,
		T *	sx,
		Sync &	sync )

inlinestaticconstexprnoexcept

One step of the backward substitution.

Every thread computes the same x_i. The owner of entry i keeps it in its register instead of writing sB[i], which every thread reads in this very barrier interval: the write MAGMA does here is a data race as soon as the threads leave lockstep.

Template Parameters

Sync	callable type of the barrier

Parameters

[in]	i	position (and solution entry) of this step
[in]	tx	rank of the calling thread in its group
[in]	rA	factored register rows of the thread
[in,out]	rB	solution registers of the thread: entry tx + K * threads_per_system in slot K
[in]	rowid	position of each row of the thread in the pivoted order
[in,out]	sB	workspace: right-hand side being reduced
[in]	sx	workspace: column i of U
[in]	sync	barrier of the group

◆ below()

template<typename T , int N, CooperativeLUppConfig< T > Config = CooperativeLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool tdls::CooperativeLUppSolverStatic< T, N, Config >::below	(	const int	tx,
		const int	K,
		const int	i )

inlinestaticconstexprnoexcept

Whether slot K of the calling thread holds a real row that may sit below column i: the slots a pivot row below the diagonal can come from, a compile-time choice in an unrolled column sweep but for the rank test of a mixed slot.

Parameters

[in]	tx	rank of the calling thread in its group
[in]	K	row slot
[in]	i	column of the step

Returns: true when slot K may hold a row below position i

◆ eliminate()

template<typename T , int N, CooperativeLUppConfig< T > Config = CooperativeLUppConfig<T>{}>

template<bool fuse_rhs, typename Sync >

TDLS_EXEC_CHECK_DISABLE TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr int tdls::CooperativeLUppSolverStatic< T, N, Config >::eliminate	(	const int	tx,
		T(&)	rA[rows_per_thread][N],
		T(&)	rB[rows_per_thread],
		int(&)	rpiv[rows_per_thread],
		T *	work,
		Sync &	sync )

inlinestaticnodiscardconstexprnoexcept

Factorization of the register rows of the group.

Template Parameters

fuse_rhs	apply the forward pass to the right-hand side as well (solve_inplace)
Sync	callable type of the barrier

Parameters

[in]	tx	rank of the calling thread in its group
[in,out]	rA	register rows of the thread
[in,out]	rB	right-hand-side registers (fuse_rhs only)
[in,out]	rpiv	pivot entry of each slot of the thread, initialized by init_piv
[in]	work	workspace of the group
[in]	sync	barrier of the group

Returns: 0, or 1 + the first column whose best pivot fell below singular_floor (MAGMA's linfo)

◆ factorize()

template<typename T , int N, CooperativeLUppConfig< T > Config = CooperativeLUppConfig<T>{}>

template<bool internal_piv, bool internal_matrix, int A_static_stride = dynamic_stride, int piv_static_stride = dynamic_stride, typename Sync = tdls::AutoSync>

TDLS_EXEC_CHECK_DISABLE TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool tdls::CooperativeLUppSolverStatic< T, N, Config >::factorize	(	const int	tx,
		T *TDLS_RESTRICT	A,
		const int	A_stride,
		int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		T *	work,
		Sync &&	sync = Sync{} )

inlinestaticnodiscardconstexprnoexcept

In-place LU factorization with partial pivoting.

Template Parameters

internal_piv	residency of piv
internal_matrix	residency of A
A_static_stride	compile-time stride of A, used instead of A_stride; tdls::dynamic_stride (the default) reads A_stride
piv_static_stride	compile-time stride of piv, used instead of piv_stride; tdls::dynamic_stride (the default) reads piv_stride
Sync	callable type of the barrier

Parameters

[in]	tx	rank of the calling thread in its group
[in,out]	A	matrix, or slice of the thread, pre-offset by the caller; its factorization on exit
[in]	A_stride	element stride of A (external mode)
[out]	piv	pivot entries, or slice of the thread (always caller-provided)
[in]	piv_stride	element stride of piv (external mode)
[in]	work	workspace of workspace_size elements, shared by the group
[in]	sync	barrier of the group, deduced by default (see tdls::AutoSync)

Returns: false on a singular matrix (factorization unspecified).

◆ forward_pass()

template<typename T , int N, CooperativeLUppConfig< T > Config = CooperativeLUppConfig<T>{}>

template<typename Sync >

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::CooperativeLUppSolverStatic< T, N, Config >::forward_pass	(	const T(&)	rA[rows_per_thread][N],
		T(&)	rB[rows_per_thread],
		const int(&)	rowid[rows_per_thread],
		T *	work,
		Sync &	sync )

inlinestaticconstexprnoexcept

Forward pass: L y = P b on the given row positions, y overwriting the right-hand-side registers.

Template Parameters

Sync	callable type of the barrier

Parameters

[in]	rA	factored register rows of the thread
[in,out]	rB	right-hand-side registers of the thread
[in]	rowid	position of each row of the thread in the pivoted order
[in]	work	workspace of the group
[in]	sync	barrier of the group

◆ forward_step()

template<typename T , int N, CooperativeLUppConfig< T > Config = CooperativeLUppConfig<T>{}>

template<typename Sync >

TDLS_EXEC_CHECK_DISABLE TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::CooperativeLUppSolverStatic< T, N, Config >::forward_step	(	const int	i,
		const T(&)	rA[rows_per_thread][N],
		T(&)	rB[rows_per_thread],
		const int(&)	rowid[rows_per_thread],
		T *	sB,
		Sync &	sync )

inlinestaticconstexprnoexcept

One step of the forward substitution.

Template Parameters

Sync	callable type of the barrier

Parameters

[in]	i	position of this step
[in]	rA	factored register rows of the thread
[in,out]	rB	right-hand-side registers of the thread
[in]	rowid	position of each row of the thread in the pivoted order
[in]	sB	workspace: broadcast slot of the pivot entry
[in]	sync	barrier of the group

◆ gather_rhs()

template<typename T , int N, CooperativeLUppConfig< T > Config = CooperativeLUppConfig<T>{}>

template<typename Sync >

TDLS_EXEC_CHECK_DISABLE TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::CooperativeLUppSolverStatic< T, N, Config >::gather_rhs	(	const int	tx,
		T(&)	rB[rows_per_thread],
		const int(&)	origin[rows_per_thread],
		T *	staged,
		Sync &	sync )

inlinestaticconstexprnoexcept

Gather of the right-hand side in the pivoted order, under physical row interchanges: each slot receives the entry of the original row it holds.

Template Parameters

Sync	callable type of the barrier

Parameters

[in]	tx	rank of the calling thread in its group
[in,out]	rB	on entry, entry tx + K * threads_per_system of the right-hand side in slot K, in original order; on exit, the entry of the row held by slot K
[in]	origin	original index of the row held by each slot
[in]	staged	workspace: the right-hand side in original order
[in]	sync	barrier of the group

◆ gather_row()

template<typename T , int N, CooperativeLUppConfig< T > Config = CooperativeLUppConfig<T>{}>

template<bool fuse_rhs>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::CooperativeLUppSolverStatic< T, N, Config >::gather_row	(	const int	tx,
		const int	i,
		const int	pos,
		const T(&)	rA[rows_per_thread][N],
		const T(&)	rB[rows_per_thread],
		const int(&)	origin[rows_per_thread],
		T *	record )

inlinestaticconstexprnoexcept

Copy the register row at position pos below column i, held by the calling thread in a slot known at run time only, to an exchange record, as save_row does.

Each entry is selected over the slots, never read through the slot index: on GPU, a register array indexed at run time is demoted to local memory, and so is one that the compiler reaches through branches it merges into such an index.

Template Parameters

fuse_rhs carry the right-hand-side entry as well

Parameters

[in]	tx	rank of the calling thread in its group
[in]	i	column of the step
[in]	pos	position of the row, below i
[in]	rA	register rows of the thread
[in]	rB	right-hand-side registers (fuse_rhs only)
[in]	origin	original index of the row held by each slot
[out]	record	exchange record, N + 2 elements

◆ init_piv()

template<typename T , int N, CooperativeLUppConfig< T > Config = CooperativeLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::CooperativeLUppSolverStatic< T, N, Config >::init_piv	(	const int	tx,
		int(&)	rpiv[rows_per_thread] )

inlinestaticconstexprnoexcept

Initial pivot entries: no row has moved yet, so the entry of each slot is the index of its row, phantom rows included (theirs stays at or above N). Under logical row interchanges the entry is the position of the row of the slot, under physical ones the original index of the row held by the slot: both start there.

Parameters

[in]	tx	rank of the calling thread in its group
[out]	rpiv	pivot entry of each slot of the thread

◆ keeps_pivot()

template<typename T , int N, CooperativeLUppConfig< T > Config = CooperativeLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool tdls::CooperativeLUppSolverStatic< T, N, Config >::keeps_pivot	(	const T	in_place,
		const T	column_max )

inlinestaticconstexprnoexcept

Relative threshold of the pivot choice: whether the row in place keeps the pivot (see CooperativeLUppConfig::relative_pivot_threshold).

Parameters

[in]	in_place	magnitude of the row in place in the column
[in]	column_max	largest magnitude of the column

Returns: true when the row in place keeps the pivot

◆ load_piv()

template<typename T , int N, CooperativeLUppConfig< T > Config = CooperativeLUppConfig<T>{}>

template<bool internal_piv>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::CooperativeLUppSolverStatic< T, N, Config >::load_piv	(	const int	tx,
		const int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		int(&)	rpiv[rows_per_thread] )

inlinestaticconstexprnoexcept

Load the pivot entries recorded by factorize.

Template Parameters

internal_piv residency of piv

Parameters

[in]	tx	rank of the calling thread in its group
[in]	piv	pivot entries, or slice of the thread (caller-pre-offset)
[in]	piv_stride	element stride of piv (external mode)
[out]	rpiv	pivot entry of each slot of the thread

◆ load_rhs()

template<typename T , int N, CooperativeLUppConfig< T > Config = CooperativeLUppConfig<T>{}>

template<bool internal_rhs>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::CooperativeLUppSolverStatic< T, N, Config >::load_rhs	(	const int	tx,
		const T *TDLS_RESTRICT	b,
		const int	rhs_stride,
		T(&)	rB[rows_per_thread] )

inlinestaticconstexprnoexcept

Load the right-hand-side entries of the rows of the calling thread.

Template Parameters

internal_rhs residency of b

Parameters

[in]	tx	rank of the calling thread in its group
[in]	b	right-hand side, or slice of the thread (caller-pre-offset)
[in]	rhs_stride	element stride of b (external mode)
[out]	rB	right-hand-side registers of the thread

◆ load_row()

template<typename T , int N, CooperativeLUppConfig< T > Config = CooperativeLUppConfig<T>{}>

template<bool fuse_rhs>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::CooperativeLUppSolverStatic< T, N, Config >::load_row	(	T(&)	row[N],
		T &	rhs,
		int &	origin,
		const T *	record )

inlinestaticconstexprnoexcept

Copy an exchange record of the workspace to a register row, the converse of save_row.

Template Parameters

fuse_rhs carry the right-hand-side entry as well

Parameters

[out]	row	register row
[out]	rhs	its right-hand-side entry (fuse_rhs only)
[out]	origin	its original index
[in]	record	exchange record, N + 2 elements

◆ load_rows()

template<typename T , int N, CooperativeLUppConfig< T > Config = CooperativeLUppConfig<T>{}>

template<bool internal_matrix>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::CooperativeLUppSolverStatic< T, N, Config >::load_rows	(	const int	tx,
		const T *TDLS_RESTRICT	A,
		const int	A_stride,
		T(&)	rA[rows_per_thread][N] )

inlinestaticconstexprnoexcept

Load the rows of the calling thread into registers.

Template Parameters

internal_matrix residency of A

Parameters

[in]	tx	rank of the calling thread in its group
[in]	A	matrix, or slice of the thread (caller-pre-offset)
[in]	A_stride	element stride of A (external mode)
[out]	rA	register rows of the thread

◆ logical_elimination_step()

template<typename T , int N, CooperativeLUppConfig< T > Config = CooperativeLUppConfig<T>{}>

template<bool fuse_rhs, typename Sync >

TDLS_EXEC_CHECK_DISABLE TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::CooperativeLUppSolverStatic< T, N, Config >::logical_elimination_step	(	const int	tx,
		const int	i,
		T(&)	rA[rows_per_thread][N],
		T(&)	rB[rows_per_thread],
		int(&)	rowid[rows_per_thread],
		T *	sB,
		T *	sx,
		T *	dsx,
		int &	linfo,
		Sync &	sync )

inlinestaticconstexprnoexcept

One column of the factorization under logical row interchanges.

With fuse_rhs, the step applies the forward pass to the right-hand side as well. Solve S x = b (column-major): the owner of the pivot row publishes its entry y_i and the rows below subtract L(r, i) y_i, as in MAGMA. Solve S^T x = b (row-major), U^T z = b: the owner of entry i publishes b'(i), final at this step, every thread computes z_i = b'(i) / U(i, i) through the reciprocal of the pivot, the owner of entry i keeps z_i in its place, and each entry c > i subtracts U(i, c) z_i, row i of U being the published pivot row.

Template Parameters

fuse_rhs	apply the forward pass to the right-hand side as well (solve_inplace)
Sync	callable type of the barrier

Parameters

[in]	tx	rank of the calling thread in its group
[in]	i	column of this step
[in,out]	rA	register rows of the thread
[in,out]	rB	right-hand-side registers (fuse_rhs only)
[in,out]	rowid	position of each row of the thread in the pivoted order
[in]	sB	workspace: broadcast slot of the right-hand-side entry of the step (fuse_rhs only)
[in]	sx	workspace: pivot row
[in]	dsx	workspace: magnitudes of the column
[in,out]	linfo	0, or 1 + the first column whose best pivot fell below singular_floor
[in]	sync	barrier of the group

◆ make_sync()

template<typename T , int N, CooperativeLUppConfig< T > Config = CooperativeLUppConfig<T>{}>

TDLS_EXEC_CHECK_DISABLE TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr auto tdls::CooperativeLUppSolverStatic< T, N, Config >::make_sync ( T * work )

inlinestaticconstexprnoexcept

The barrier deduced for the group of the calling thread.

The entry points deduce it on every call when no barrier is passed. A caller that also needs the barrier for its own exchanges between the threads of the group builds it once with this function, calls it at will, and passes it to the entry points, which then use it as is. On GPU, every thread of the group must call this function together, as the entry points (see tdls::AutoSync).

Parameters

[in] work workspace of the group

Returns: tdls::NoSync for one thread per system, the deduced barrier otherwise

◆ physical_elimination_step()

template<typename T , int N, CooperativeLUppConfig< T > Config = CooperativeLUppConfig<T>{}>

template<bool fuse_rhs, typename Sync >

TDLS_EXEC_CHECK_DISABLE TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::CooperativeLUppSolverStatic< T, N, Config >::physical_elimination_step	(	const int	tx,
		const int	i,
		T(&)	rA[rows_per_thread][N],
		T(&)	rB[rows_per_thread],
		int(&)	origin[rows_per_thread],
		T *	work,
		int &	linfo,
		Sync &	sync )

inlinestaticconstexprnoexcept

One column of the factorization under physical row interchanges.

The row at position k lives in slot k / threads_per_system of thread k % threads_per_system. Each thread publishes its candidate: the largest magnitude of column i among its rows at or below position i, the first one on ties, with its position. Every thread finds the same column maximum and the same pivot as the logical scan, ties and NaN included: the row in place when it reaches the maximum, the first position holding it otherwise. The owner of position i publishes its row whole. When the pivot is not in place, the owner of the pivot row publishes it as well, and both rows are exchanged before the elimination, their multipliers included: the factored rows end in the pivoted order, as in LAPACK.

With fuse_rhs, the forward pass of the right-hand side is that of the logical scheme on positions. Solve S x = b: the entry of each row travels with it. Solve S^T x = b: the right-hand side is indexed by the columns of S, which never move; the owner of position i is the owner of entry i, publishes b'(i) with its row and keeps z_i in its place.

Template Parameters

fuse_rhs	apply the forward pass to the right-hand side as well (solve_inplace)
Sync	callable type of the barrier

Parameters

[in]	tx	rank of the calling thread in its group
[in]	i	column of this step
[in,out]	rA	register rows of the thread
[in,out]	rB	right-hand-side registers (fuse_rhs only)
[in,out]	origin	original index of the row held by each slot
[in]	work	workspace of the group
[in,out]	linfo	0, or 1 + the first column whose best pivot fell below singular_floor
[in]	sync	barrier of the group

◆ save_row()

template<typename T , int N, CooperativeLUppConfig< T > Config = CooperativeLUppConfig<T>{}>

template<bool fuse_rhs>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::CooperativeLUppSolverStatic< T, N, Config >::save_row	(	const T(&)	row[N],
		const T &	rhs,
		const int	origin,
		T *	record )

inlinestaticconstexprnoexcept

Copy a register row to an exchange record of the workspace: its N entries, its right-hand-side entry (fuse_rhs only), then its original index. The index travels as a value of T, exact since the offset bounds keep N far below the mantissa range of every scalar type.

Template Parameters

fuse_rhs carry the right-hand-side entry as well

Parameters

[in]	row	register row
[in]	rhs	its right-hand-side entry (fuse_rhs only)
[in]	origin	its original index
[out]	record	exchange record, N + 2 elements

◆ scatter_row()

template<typename T , int N, CooperativeLUppConfig< T > Config = CooperativeLUppConfig<T>{}>

template<bool fuse_rhs>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::CooperativeLUppSolverStatic< T, N, Config >::scatter_row	(	const int	tx,
		const int	i,
		const int	pos,
		T(&)	rA[rows_per_thread][N],
		T(&)	rB[rows_per_thread],
		int(&)	origin[rows_per_thread],
		const T *	record )

inlinestaticconstexprnoexcept

Copy an exchange record to the register row at position pos below column i, held by the calling thread in a slot known at run time only, the converse of gather_row: each slot takes the record or keeps its row through a select.

Template Parameters

fuse_rhs carry the right-hand-side entry as well

Parameters

[in]	tx	rank of the calling thread in its group
[in]	i	column of the step
[in]	pos	position of the row, below i
[in,out]	rA	register rows of the thread
[in,out]	rB	right-hand-side registers (fuse_rhs only)
[in,out]	origin	original index of the row held by each slot
[in]	record	exchange record, N + 2 elements

◆ segment()

template<typename T , int N, CooperativeLUppConfig< T > Config = CooperativeLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr T * tdls::CooperativeLUppSolverStatic< T, N, Config >::segment	(	T *	work,
		const int	step )

inlinestaticconstexprnoexcept

Half of the double buffer of the transposed substitutions used by a step: N + 1 elements, a row segment and one value. A thread reads the half of step s after the barrier of step s; the next write to that half comes at step s + 2, after the barrier of step s + 1, which every thread reaches only once its reads of step s are done. Both halves take 2 * N + 2 elements, within the workspace for N >= 2; a dimension of 1 uses the first half only.

Parameters

[in]	work	workspace of the group
[in]	step	index of the step

Returns: the half of the step

◆ slot_has_rows()

template<typename T , int N, CooperativeLUppConfig< T > Config = CooperativeLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool tdls::CooperativeLUppSolverStatic< T, N, Config >::slot_has_rows ( const int K )

inlinestaticconstexprnoexcept

Whether slot K holds a real row for at least one thread.

Parameters

[in] K row slot

Returns: true when some thread of the group has a real row in slot K

◆ slot_is_full()

template<typename T , int N, CooperativeLUppConfig< T > Config = CooperativeLUppConfig<T>{}>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool tdls::CooperativeLUppSolverStatic< T, N, Config >::slot_is_full ( const int K )

inlinestaticconstexprnoexcept

Whether slot K holds a real row for every thread.

Parameters

[in] K row slot

Returns: true when every thread of the group has a real row in slot K

◆ solve()

template<typename T , int N, CooperativeLUppConfig< T > Config = CooperativeLUppConfig<T>{}>

template<bool internal_rhs, bool internal_piv, bool internal_matrix, int A_static_stride = dynamic_stride, int piv_static_stride = dynamic_stride, int rhs_static_stride = dynamic_stride, typename Sync = tdls::AutoSync>

TDLS_EXEC_CHECK_DISABLE TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool tdls::CooperativeLUppSolverStatic< T, N, Config >::solve	(	const int	tx,
		T *TDLS_RESTRICT	A,
		const int	A_stride,
		int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		const T *TDLS_RESTRICT	b,
		T *TDLS_RESTRICT	x,
		const int	rhs_stride,
		T *	work,
		Sync &&	sync = Sync{} )

inlinestaticnodiscardconstexprnoexcept

factorize + substitute in one call. The rows stay in registers between the two, so the factors are stored once and never reloaded.

The barrier sequence does not depend on the verdict: a singular matrix is substituted as well, and x is then unspecified.

Template Parameters

internal_rhs	residency of b and x
internal_piv	residency of piv
internal_matrix	residency of A
A_static_stride	compile-time stride of A, used instead of A_stride; tdls::dynamic_stride (the default) reads A_stride
piv_static_stride	compile-time stride of piv, used instead of piv_stride; tdls::dynamic_stride (the default) reads piv_stride
rhs_static_stride	compile-time stride of b and x, used instead of rhs_stride; tdls::dynamic_stride (the default) reads rhs_stride
Sync	callable type of the barrier

Parameters

[in]	tx	rank of the calling thread in its group
[in,out]	A	on entry the matrix, or slice of the thread, pre-offset by the caller; on exit its factorization (usable for further substitute* calls)
[in]	A_stride	element stride of A (external mode)
[out]	piv	pivot entries, or slice of the thread (always caller-provided)
[in]	piv_stride	element stride of piv (external mode)
[in]	b	right-hand side, in original order
[out]	x	solution
[in]	rhs_stride	element stride of b and x (external mode)
[in]	work	workspace of workspace_size elements, shared by the group
[in]	sync	barrier of the group, deduced by default (see tdls::AutoSync)

Returns: false on a singular matrix (x unspecified).

◆ solve_inplace()

template<typename T , int N, CooperativeLUppConfig< T > Config = CooperativeLUppConfig<T>{}>

template<bool internal_rhs, bool internal_piv, bool internal_matrix, int A_static_stride = dynamic_stride, int piv_static_stride = dynamic_stride, int rhs_static_stride = dynamic_stride, typename Sync = tdls::AutoSync>

TDLS_EXEC_CHECK_DISABLE TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr bool tdls::CooperativeLUppSolverStatic< T, N, Config >::solve_inplace	(	const int	tx,
		T *TDLS_RESTRICT	A,
		const int	A_stride,
		int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		T *TDLS_RESTRICT	y,
		const int	rhs_stride,
		T *	work,
		Sync &&	sync = Sync{} )

inlinestaticnodiscardconstexprnoexcept

Factorization with the forward substitution folded in: the original MAGMA kernel.

y holds the unpermuted right-hand side on entry and the solution on exit. The elimination is applied to y while the factorization runs, so the separate forward pass disappears and only the backward pass remains. The result is bitwise-identical to factorize + substitute_inplace. The barrier sequence does not depend on the verdict: a singular matrix is substituted as well, and y is then unspecified.

Template Parameters

internal_rhs	residency of y
internal_piv	residency of piv
internal_matrix	residency of A
A_static_stride	compile-time stride of A, used instead of A_stride; tdls::dynamic_stride (the default) reads A_stride
piv_static_stride	compile-time stride of piv, used instead of piv_stride; tdls::dynamic_stride (the default) reads piv_stride
rhs_static_stride	compile-time stride of y, used instead of rhs_stride; tdls::dynamic_stride (the default) reads rhs_stride
Sync	callable type of the barrier

Parameters

[in]	tx	rank of the calling thread in its group
[in,out]	A	on entry the matrix, or slice of the thread, pre-offset by the caller; on exit its factorization
[in]	A_stride	element stride of A (external mode)
[out]	piv	pivot entries, or slice of the thread (always caller-provided)
[in]	piv_stride	element stride of piv (external mode)
[in,out]	y	right-hand side on entry, solution on exit
[in]	rhs_stride	element stride of y (external mode)
[in]	work	workspace of workspace_size elements, shared by the group
[in]	sync	barrier of the group, deduced by default (see tdls::AutoSync)

Returns: false on a singular matrix (y unspecified).

◆ store_piv()

template<typename T , int N, CooperativeLUppConfig< T > Config = CooperativeLUppConfig<T>{}>

template<bool internal_piv>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::CooperativeLUppSolverStatic< T, N, Config >::store_piv	(	const int	tx,
		int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		const int(&)	rpiv[rows_per_thread] )

inlinestaticconstexprnoexcept

Record the pivot entries of the calling thread.

Template Parameters

internal_piv residency of piv

Parameters

[in]	tx	rank of the calling thread in its group
[out]	piv	pivot entries, or slice of the thread (caller-pre-offset)
[in]	piv_stride	element stride of piv (external mode)
[in]	rpiv	pivot entry of each slot of the thread

◆ store_rows()

template<typename T , int N, CooperativeLUppConfig< T > Config = CooperativeLUppConfig<T>{}>

template<bool internal_matrix>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::CooperativeLUppSolverStatic< T, N, Config >::store_rows	(	const int	tx,
		T *TDLS_RESTRICT	A,
		const int	A_stride,
		const T(&)	rA[rows_per_thread][N] )

inlinestaticconstexprnoexcept

Store the register rows of the calling thread.

Template Parameters

internal_matrix residency of A

Parameters

[in]	tx	rank of the calling thread in its group
[in,out]	A	matrix, or slice of the thread (caller-pre-offset)
[in]	A_stride	element stride of A (external mode)
[in]	rA	register rows of the thread

◆ store_solution()

template<typename T , int N, CooperativeLUppConfig< T > Config = CooperativeLUppConfig<T>{}>

template<bool internal_rhs>

TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::CooperativeLUppSolverStatic< T, N, Config >::store_solution	(	const int	tx,
		T *TDLS_RESTRICT	x,
		const int	rhs_stride,
		const T(&)	rB[rows_per_thread] )

inlinestaticconstexprnoexcept

Store the solution entries held by the calling thread.

Template Parameters

internal_rhs residency of x

Parameters

[in]	tx	rank of the calling thread in its group
[out]	x	solution, or slice of the thread (caller-pre-offset)
[in]	rhs_stride	element stride of x (external mode)
[in]	rB	solution registers of the thread

◆ substitute()

template<typename T , int N, CooperativeLUppConfig< T > Config = CooperativeLUppConfig<T>{}>

template<bool internal_rhs, bool internal_piv, bool internal_matrix, int A_static_stride = dynamic_stride, int piv_static_stride = dynamic_stride, int rhs_static_stride = dynamic_stride, typename Sync = tdls::AutoSync>

TDLS_EXEC_CHECK_DISABLE TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::CooperativeLUppSolverStatic< T, N, Config >::substitute	(	const int	tx,
		const T *TDLS_RESTRICT	A,
		const int	A_stride,
		const int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		const T *TDLS_RESTRICT	b,
		T *TDLS_RESTRICT	x,
		const int	rhs_stride,
		T *	work,
		Sync &&	sync = Sync{} )

inlinestaticconstexprnoexcept

Solve from a factorization produced by factorize (b and x distinct).

Template Parameters

internal_rhs	residency of b and x
internal_piv	residency of piv
internal_matrix	residency of A
A_static_stride	compile-time stride of A, used instead of A_stride; tdls::dynamic_stride (the default) reads A_stride
piv_static_stride	compile-time stride of piv, used instead of piv_stride; tdls::dynamic_stride (the default) reads piv_stride
rhs_static_stride	compile-time stride of b and x, used instead of rhs_stride; tdls::dynamic_stride (the default) reads rhs_stride
Sync	callable type of the barrier

Parameters

[in]	tx	rank of the calling thread in its group
[in]	A	factored matrix, or slice of the thread
[in]	A_stride	element stride of A (external mode)
[in]	piv	pivot entries produced by factorize
[in]	piv_stride	element stride of piv (external mode)
[in]	b	right-hand side, in original order
[out]	x	solution
[in]	rhs_stride	element stride of b and x (external mode)
[in]	work	workspace of workspace_size elements, shared by the group
[in]	sync	barrier of the group, deduced by default (see tdls::AutoSync)

◆ substitute_canonical()

template<typename T , int N, CooperativeLUppConfig< T > Config = CooperativeLUppConfig<T>{}>

template<bool internal_rhs, bool internal_piv, bool internal_matrix, int A_static_stride = dynamic_stride, int piv_static_stride = dynamic_stride, int rhs_static_stride = dynamic_stride, typename Sync = tdls::AutoSync>

TDLS_EXEC_CHECK_DISABLE TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::CooperativeLUppSolverStatic< T, N, Config >::substitute_canonical	(	const int	tx,
		const T *TDLS_RESTRICT	A,
		const int	A_stride,
		const int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		const int	col,
		T *TDLS_RESTRICT	x,
		const int	rhs_stride,
		T *	work,
		Sync &&	sync = Sync{} )

inlinestaticconstexprnoexcept

Solve with b = e_col generated on the fly: the consistent-tangent-operator path.

Template Parameters

internal_rhs	residency of x
internal_piv	residency of piv
internal_matrix	residency of A
A_static_stride	compile-time stride of A, used instead of A_stride; tdls::dynamic_stride (the default) reads A_stride
piv_static_stride	compile-time stride of piv, used instead of piv_stride; tdls::dynamic_stride (the default) reads piv_stride
rhs_static_stride	compile-time stride of x, used instead of rhs_stride; tdls::dynamic_stride (the default) reads rhs_stride
Sync	callable type of the barrier

Parameters

[in]	tx	rank of the calling thread in its group
[in]	A	factored matrix, or slice of the thread
[in]	A_stride	element stride of A (external mode)
[in]	piv	pivot entries produced by factorize
[in]	piv_stride	element stride of piv (external mode)
[in]	col	index of the canonical column e_col
[out]	x	solution
[in]	rhs_stride	element stride of x (external mode)
[in]	work	workspace of workspace_size elements, shared by the group
[in]	sync	barrier of the group, deduced by default (see tdls::AutoSync)

◆ substitute_inplace()

template<typename T , int N, CooperativeLUppConfig< T > Config = CooperativeLUppConfig<T>{}>

template<bool internal_rhs, bool internal_piv, bool internal_matrix, int A_static_stride = dynamic_stride, int piv_static_stride = dynamic_stride, int rhs_static_stride = dynamic_stride, typename Sync = tdls::AutoSync>

TDLS_EXEC_CHECK_DISABLE TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::CooperativeLUppSolverStatic< T, N, Config >::substitute_inplace	(	const int	tx,
		const T *TDLS_RESTRICT	A,
		const int	A_stride,
		const int *TDLS_RESTRICT	piv,
		const int	piv_stride,
		T *TDLS_RESTRICT	x,
		const int	rhs_stride,
		T *	work,
		Sync &&	sync = Sync{} )

inlinestaticconstexprnoexcept

Solve in place from a factorization produced by factorize: x holds the right-hand side on entry and the solution on exit.

Each thread reads and writes only the entries of its own rows, the same set before and after, so the in-place form needs no copy.

Template Parameters

internal_rhs	residency of x
internal_piv	residency of piv
internal_matrix	residency of A
A_static_stride	compile-time stride of A, used instead of A_stride; tdls::dynamic_stride (the default) reads A_stride
piv_static_stride	compile-time stride of piv, used instead of piv_stride; tdls::dynamic_stride (the default) reads piv_stride
rhs_static_stride	compile-time stride of x, used instead of rhs_stride; tdls::dynamic_stride (the default) reads rhs_stride
Sync	callable type of the barrier

Parameters

[in]	tx	rank of the calling thread in its group
[in]	A	factored matrix, or slice of the thread
[in]	A_stride	element stride of A (external mode)
[in]	piv	pivot entries produced by factorize
[in]	piv_stride	element stride of piv (external mode)
[in,out]	x	right-hand side on entry, solution on exit
[in]	rhs_stride	element stride of x (external mode)
[in]	work	workspace of workspace_size elements, shared by the group
[in]	sync	barrier of the group, deduced by default (see tdls::AutoSync)

◆ substitution()

template<typename T , int N, CooperativeLUppConfig< T > Config = CooperativeLUppConfig<T>{}>

template<bool backward_only, typename Sync >

TDLS_EXEC_CHECK_DISABLE TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::CooperativeLUppSolverStatic< T, N, Config >::substitution	(	const int	tx,
		const T(&)	rA[rows_per_thread][N],
		T(&)	rB[rows_per_thread],
		const int(&)	rpiv[rows_per_thread],
		T *	work,
		Sync &	sync )

inlinestaticconstexprnoexcept

Substitution from the factored rows, the right-hand side in the registers on entry, the solution entries of the thread on exit: L y = P b then U x = y under the column-major layout, U^T z = b, L^T w = z and x = P^T w under the row-major one.

Template Parameters

backward_only	skip the forward pass, folded into the factorization (solve_inplace)
Sync	callable type of the barrier

Parameters

[in]	tx	rank of the calling thread in its group
[in]	rA	factored register rows of the thread
[in,out]	rB	right-hand-side registers of the thread on entry (forward-reduced with backward_only), solution entries tx + K * threads_per_system on exit
[in]	rpiv	pivot entry of each slot of the thread
[in]	work	workspace of the group
[in]	sync	barrier of the group

◆ transposed_backward_step()

template<typename T , int N, CooperativeLUppConfig< T > Config = CooperativeLUppConfig<T>{}>

template<typename Sync >

TDLS_EXEC_CHECK_DISABLE TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::CooperativeLUppSolverStatic< T, N, Config >::transposed_backward_step	(	const int	tx,
		const int	k,
		const T(&)	rA[rows_per_thread][N],
		T(&)	rB[rows_per_thread],
		const int(&)	rowid[rows_per_thread],
		T *	work,
		Sync &	sync )

inlinestaticconstexprnoexcept

One step of the backward substitution L^T w = z.

The step uses the half of step k + 1: the first step, k = N - 1, then writes the half that the last step of the forward pass does not read, so that no barrier separates the passes.

Template Parameters

Sync	callable type of the barrier

Parameters

[in]	tx	rank of the calling thread in its group
[in]	k	position of this step
[in]	rA	factored register rows of the thread
[in,out]	rB	entries tx + K * threads_per_system: z below position k, w from it on
[in]	rowid	position of each row of the thread in the pivoted order
[in]	work	workspace of the group
[in]	sync	barrier of the group

◆ transposed_forward_step()

template<typename T , int N, CooperativeLUppConfig< T > Config = CooperativeLUppConfig<T>{}>

template<typename Sync >

TDLS_EXEC_CHECK_DISABLE TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::CooperativeLUppSolverStatic< T, N, Config >::transposed_forward_step	(	const int	tx,
		const int	i,
		const T(&)	rA[rows_per_thread][N],
		T(&)	rB[rows_per_thread],
		const int(&)	rowid[rows_per_thread],
		T *	work,
		Sync &	sync )

inlinestaticconstexprnoexcept

One step of the forward substitution U^T z = b.

Template Parameters

Sync	callable type of the barrier

Parameters

[in]	tx	rank of the calling thread in its group
[in]	i	position of this step
[in]	rA	factored register rows of the thread
[in,out]	rB	entries tx + K * threads_per_system: b' below position i, z from it on
[in]	rowid	position of each row of the thread in the pivoted order
[in]	work	workspace of the group
[in]	sync	barrier of the group

◆ transposed_passes()

template<typename T , int N, CooperativeLUppConfig< T > Config = CooperativeLUppConfig<T>{}>

template<bool backward_only, typename Sync >

TDLS_EXEC_CHECK_DISABLE TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::CooperativeLUppSolverStatic< T, N, Config >::transposed_passes	(	const int	tx,
		const T(&)	rA[rows_per_thread][N],
		T(&)	rB[rows_per_thread],
		const int(&)	rowid[rows_per_thread],
		T *	work,
		Sync &	sync )

inlinestaticconstexprnoexcept

Forward then backward pass of S^T x = b on the given row positions: b in the entries of the thread on entry, w at the entries of the same positions on exit. Position 0 updates no entry in the backward pass: its step is skipped.

Template Parameters

backward_only	skip the forward pass, folded into the factorization (solve_inplace)
Sync	callable type of the barrier

Parameters

[in]	tx	rank of the calling thread in its group
[in]	rA	factored register rows of the thread
[in,out]	rB	entries tx + K * threads_per_system
[in]	rowid	position of each row of the thread in the pivoted order
[in]	work	workspace of the group
[in]	sync	barrier of the group

◆ unpermute_solution()

template<typename T , int N, CooperativeLUppConfig< T > Config = CooperativeLUppConfig<T>{}>

template<typename Sync >

TDLS_EXEC_CHECK_DISABLE TDLS_HOST_DEVICE static TDLS_FORCE_INLINE_GPU constexpr void tdls::CooperativeLUppSolverStatic< T, N, Config >::unpermute_solution	(	const int	tx,
		T(&)	rB[rows_per_thread],
		const int(&)	rpiv[rows_per_thread],
		T *	work,
		Sync &	sync )

inlinestaticconstexprnoexcept

x = P^T w, through the workspace. Logical row interchanges: the slot of row r of S takes w at the position of the row. Physical ones: w_p goes to the original index of the row at position p. The exchange is staged in the half of the double buffer that the last backward step does not read.

Template Parameters

Sync	callable type of the barrier

Parameters

[in]	tx	rank of the calling thread in its group
[in,out]	rB	w of the positions tx + K * threads_per_system on entry, solution entries tx + K * threads_per_system on exit
[in]	rpiv	pivot entry of each slot of the thread
[in]	work	workspace of the group
[in]	sync	barrier of the group

Member Data Documentation

◆ rows_per_thread

template<typename T , int N, CooperativeLUppConfig< T > Config = CooperativeLUppConfig<T>{}>

int tdls::CooperativeLUppSolverStatic< T, N, Config >::rows_per_thread

staticconstexpr

Initial value:

= Config.rows_per_thread < 1   ? 1
                                           : Config.rows_per_thread < N ? Config.rows_per_thread
                                                                        : N

Rows held by each thread: Config.rows_per_thread, or N when it is larger. A value below 1, rejected above, is clamped to 1 so that the rejection stays the only diagnostic.

◆ workspace_size

template<typename T , int N, CooperativeLUppConfig< T > Config = CooperativeLUppConfig<T>{}>

int tdls::CooperativeLUppSolverStatic< T, N, Config >::workspace_size

staticconstexpr

Initial value:

=
        detail::barrier_elements + (Config.row_interchange == RowInterchange::Logical
                                        ? 3 * N
                                        : 2 * N + 2 * threads_per_system + 5)

Elements of the workspace shared by the threads of a group: the two elements of the deduced CPU barrier, then 3 * N under logical row interchanges, 2 * N + 2 * threads_per_system + 5 under physical ones.

The documentation for this struct was generated from the following file:

include/tdls/solvers/cooperative_lupp/solver_static.hpp

Static Public Member Functions

Static Public Attributes

Detailed Description

Member Function Documentation

◆ at_position()

◆ backward_pass()

◆ backward_step()

◆ below()

◆ eliminate()

◆ factorize()

◆ forward_pass()

◆ forward_step()

◆ gather_rhs()

◆ gather_row()

◆ init_piv()

◆ keeps_pivot()

◆ load_piv()

◆ load_rhs()

◆ load_row()

◆ load_rows()

◆ logical_elimination_step()

◆ make_sync()

◆ physical_elimination_step()

◆ save_row()

◆ scatter_row()

◆ segment()

◆ slot_has_rows()

◆ slot_is_full()

◆ solve()

◆ solve_inplace()

◆ store_piv()

◆ store_rows()

◆ store_solution()

◆ substitute()

◆ substitute_canonical()

◆ substitute_inplace()

◆ substitution()

◆ transposed_backward_step()

◆ transposed_forward_step()

◆ transposed_passes()

◆ unpermute_solution()

Member Data Documentation

◆ rows_per_thread

◆ workspace_size