/*-
 * Copyright (c) 1982, 1988, 1991, 1993
 *	The Regents of the University of California.  All rights reserved.
 * (c) UNIX System Laboratories, Inc.
 * All or some portions of this file are derived from material licensed
 * to the University of California by American Telephone and Telegraph
 * Co. or Unix System Laboratories, Inc. and are reproduced herein with
 * the permission of UNIX System Laboratories, Inc.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 *	@(#)systm.h	8.7 (Berkeley) 3/29/95
 * $FreeBSD: src/sys/sys/systm.h,v 1.111.2.18 2002/12/17 18:04:02 sam Exp $
 */

#ifndef _SYS_SYSTM_H_
#define	_SYS_SYSTM_H_

#ifndef _KERNEL
#error "This file should not be included by userland programs."
#endif

#ifndef _MACHINE_TYPES_H_
#include <machine/types.h>
#endif
#include <machine/stdarg.h>
#include <machine/atomic.h>
#include <machine/cpufunc.h>
#include <machine/int_limits.h>

extern int securelevel;		/* system security level (see init(8)) */
extern int kernel_mem_readonly;	/* disable writes to kernel memory */

extern int cold;		/* nonzero if we are doing a cold boot */
extern int tsleep_now_works;	/* tsleep won't just return any more */
extern const char *panicstr;	/* panic message */
extern int dumping;		/* system is dumping */
extern int safepri;		/* safe ipl when cold or panicing */
extern int osreldate;		/* System release date */
extern char version[];		/* system version */
extern char copyright[];	/* system copyright */

extern u_char curpriority;	/* priority of current process */
extern int cpu_mwait_spin;	/* typically set in machdep, used by lwkt */

extern long physmem;		/* physical memory */

extern cdev_t dumpdev;		/* dump device */
extern u_int64_t dumplo64;	/* block number into dumpdev, start of dump */

extern cdev_t rootdev;		/* root device */
extern cdev_t rootdevs[2];	/* possible root devices */
extern char *rootdevnames[2];	/* names of possible root devices */
extern char *ZeroPage;

extern int boothowto;		/* reboot flags, from console subsystem */
extern int bootverbose;		/* nonzero to print verbose messages */

extern int maxusers;		/* system tune hint */

extern int ncpus;		/* total number of cpus (real, hyper, virtual)*/
extern int ncpus_fit;		/* round up to a power of 2 */
extern int ncpus_fit_mask;	/* ncpus_fit - 1 */
extern int clocks_running;	/* timing/timeout subsystem is operational */

extern u_int cpu_mi_feature;	/* CPU_MI_XXX machine-nonspecific features */
extern cpumask_t usched_global_cpumask;

extern int nfs_diskless_valid;	/* NFS diskless params were obtained */
extern vm_paddr_t Maxmem;	/* Highest physical memory address in system */

#ifdef	INVARIANTS		/* The option is always available */
#define	KASSERT(exp,msg)	do { if (__predict_false(!(exp)))	  \
					panic msg; } while (0)

#define	KKASSERT(exp)		do { if (__predict_false(!(exp)))	  \
					panic("assertion \"%s\" failed "  \
					"in %s at %s:%u", #exp, __func__, \
					__FILE__, __LINE__); } while (0)

#define	KKASSERT_UNSPIN(exp, spin)					  \
				do { if (__predict_false(!(exp))) { 	  \
					spin_unlock_any(spin);		  \
					panic("assertion \"%s\" failed "  \
					"in %s at %s:%u", #exp, __func__, \
					__FILE__, __LINE__); } } while (0)
#define	__debugvar
#define	__assert_unreachable()						  \
				do {					  \
					panic("executing segment marked " \
					"as unreachable at %s:%d (%s)\n", \
					__FILE__, __LINE__, __func__);    \
				} while (0)
#else
#define	KASSERT(exp,msg)		do { } while (0)
#define	KKASSERT(exp)			do { } while (0)
#define	KKASSERT_UNSPIN(exp, spin)	do { } while (0)
#define	__debugvar		__attribute__((__unused__))
#define	__assert_unreachable()	__unreachable()
#endif

/*
 * Align variables.
 */
#define	__read_mostly		__section(".data.read_mostly")
#define	__read_frequently	__section(".data.read_frequently")
#define	__exclusive_cache_line	__aligned(__VM_CACHELINE_SIZE*2)	\
				__section(".data.exclusive_cache_line")

/*
 * General function declarations.
 */

struct intrframe;
struct spinlock;
struct lock;
struct mtx;
struct lwkt_serialize;
struct malloc_type;
struct proc;
struct timeval;
struct tty;
struct uio;
struct globaldata;
struct thread;
struct trapframe;
struct user;
struct vmspace;
struct savetls;
struct krate;
struct _jmp_buf;

void	Debugger (const char *msg);
void	print_backtrace(int count);
void	mi_gdinit (struct globaldata *gd, int cpu);
void	slab_gdinit (struct globaldata *gd);
void	mi_proc0init(struct globaldata *gd, struct user *proc0paddr);
int	setjmp(struct _jmp_buf *) __returns_twice;
void	longjmp(struct _jmp_buf *, int) __dead2;
int	nullop (void);
int	seltrue (cdev_t dev, int which);
int	ureadc (int, struct uio *);
void	*hashinit (int count, struct malloc_type *type, u_long *hashmask);
void	hashdestroy(void *vhashtbl, struct malloc_type *type, u_long hashmask);
void	*phashinit (int count, struct malloc_type *type, u_long *nentries);
void	*hashinit_ext (int count, size_t size,
			struct malloc_type *type, u_long *hashmask);
void	*phashinit_ext (int count, size_t size,
			struct malloc_type *type, u_long *nentries);

int	cpu_sanitize_frame (struct trapframe *);
int	cpu_sanitize_tls (struct savetls *);
void	cpu_spinlock_contested(void);
void	cpu_halt (void) __dead2;
void	cpu_idle_halt (void);
void	cpu_reset (void);
void	cpu_boot (int);
void	cpu_rootconf (void);
void	cpu_vmspace_alloc(struct vmspace *);
void	cpu_vmspace_free(struct vmspace *);
void	cpu_vkernel_trap(struct trapframe *, int);
enum vmm_guest_type detect_virtual(void);
void	set_user_TLS(void);
void	set_vkernel_fp(struct trapframe *);
int	kvm_access_check(vm_offset_t, vm_offset_t, int);

/*
 * Old CRC32 API
 */
extern const uint32_t crc32_tab[];
uint32_t crc32(const void *buf, size_t size);
uint32_t crc32_ext(const void *buf, size_t size, uint32_t ocrc);

/*
 * Newer (fast) iscsi CRC32 APIs.
 *
 * NOTE: iscsi_crc32_ext() inverts ocrc input and output as expected,
 *	 pass a seed of 0 for the equivalent of iscsi_crc32().
 *
 *	 calculate_crc32c() does not invert crc32c input and output
 *	 to maintain FreeBSD API compatibility.
 */
uint32_t iscsi_crc32(const void *buf, size_t size);
uint32_t iscsi_crc32_ext(const void *buf, size_t size, uint32_t ocrc);
uint32_t calculate_crc32c(uint32_t crc32c, const unsigned char *buffer,
			unsigned int length);

void	init_param1 (void);
void	init_param2 (int physpages);
void	tablefull (const char *);
int	kvcprintf (char const *, void (*)(int, void*), void *,
		      __va_list) __printf0like(1, 0);
void	kvcreinitspin(void);
int	log (int, const char *, ...) __printflike(2, 3);
void	logwakeup (void);
void	log_console (struct uio *);
int	kprintf (const char *, ...) __printflike(1, 2);
void	kprintf0 (const char *, ...) __printflike(1, 2);
int	krateprintf (struct krate *, const char *, ...) __printflike(2, 3);
int	ksnprintf (char *, size_t, const char *, ...) __printflike(3, 4);
int	ksprintf (char *buf, const char *, ...) __printflike(2, 3);
int	uprintf (const char *, ...) __printflike(1, 2);
int	kvprintf (const char *, __va_list) __printflike(1, 0);
int	kvsnprintf (char *, size_t, const char *,
			__va_list) __printflike(3, 0);
int	kvasnprintf (char **, size_t, const char *,
			__va_list) __printflike(3, 0);
int	kvsprintf (char *buf, const char *,
			__va_list) __printflike(2, 0);
int	ttyprintf (struct tty *, const char *, ...) __printflike(2, 3);
void	hexdump (const void *ptr, int length, const char *hdr, int flags);
void	kprint_cpuset(cpumask_t *mask);
#define	HD_COLUMN_MASK	0xff
#define	HD_DELIM_MASK	0xff00
#define	HD_OMIT_COUNT	(1 << 16)
#define	HD_OMIT_HEX	(1 << 17)
#define	HD_OMIT_CHARS	(1 << 18)
int	ksscanf(const char *, char const *, ...)
	    __nonnull(1, 2) __scanflike(2, 3);
int	kvsscanf(const char *, char const *, __va_list)
	    __nonnull(1, 2) __scanflike(2, 0);
void	kvasfree(char **);

long	strtol(const char *, char **, int) __nonnull(1);
u_long	strtoul(const char *, char **, int) __nonnull(1);
quad_t	strtoq(const char *, char **, int) __nonnull(1);
u_quad_t strtouq(const char *, char **, int) __nonnull(1);

/*
 * NOTE: some functions commonly used by device drivers may be passed
 *       pointers to volatile storage, volatile set to avoid warnings.
 *
 * NOTE: When using builtin's, GCC does enormous optimizations and makes
 *       a number of assumptions that can cause later unrelated conditionals
 *       to be optimized out.  In the case of memset and memmove, GCC
 *       assumes that (to) and (from) cannot be NULL (even when (len) might
 *       be 0), and will optimize-out later NULL tests on (to) or (from).
 */
#if 1
#define bcopy(from, to, len)				\
	__builtin_memmove(__DEQUALIFY(void *, (to)),	\
			  __DEQUALIFY(void *, (from)),	\
			  (len))
#define memcpy(to, from, len)				\
	__builtin_memcpy(__DEQUALIFY(void *, (to)),	\
			  __DEQUALIFY(void *, (from)),	\
			  (len))
#define memset(ptr, c, len)				\
	__builtin_memset(__DEQUALIFY(void *, (ptr)), (c), (len))
#define memmove(to, from, len)				\
	__builtin_memmove(__DEQUALIFY(void *, (to)),	\
			  __DEQUALIFY(void *, (from)),	\
			  (len))
#define bzero(buf, len)					\
	__builtin_memset(__DEQUALIFY(void *, (buf)), 0, (len))
#else
void	bcopy(volatile const void *from, volatile void *to, size_t len)
	    __nonnull(1, 2);
void	*memcpy(void *to, const void *from, size_t len)
	    __nonnull(1, 2);
void	*memmove(void *, const void *, size_t);
void	*memset(void *, int, size_t);
void	bzero(volatile void *buf, size_t len) __nonnull(1);
#endif
void	_bcopy(volatile const void *from, volatile void *to, size_t len)
	    __nonnull(1, 2);
void	*_memcpy(void *to, const void *from, size_t len)
	    __nonnull(1, 2);
void	*_memmove(void *, const void *, size_t);
void	*_memset(void *, int, size_t);
void	_bzero(volatile void *buf, size_t len) __nonnull(1);
void	bzeront(volatile void *buf, size_t len) __nonnull(1);
void	explicit_bzero(void *buf, size_t len) __nonnull(1);

long	kreadmem64(const void *addr);

int	copystr (const void *kfaddr, void *kdaddr, size_t len,
		size_t *lencopied) __nonnull(1, 2);
int	copyinstr (const void *udaddr, void *kaddr, size_t len,
		size_t *lencopied) __nonnull(1, 2);
int	copyin(const void *udaddr, void *kaddr, size_t len) __nonnull(1, 2);
int	copyin_nofault(const void *udaddr, void *kaddr, size_t len)
	    __nonnull(1, 2);
int	copyout(const void *kaddr, void *udaddr, size_t len) __nonnull(1, 2);
int	copyout_nofault(const void *kaddr, void *udaddr, size_t len)
	    __nonnull(1, 2);

int	fubyte (const uint8_t *base);
int	subyte (uint8_t *base, uint8_t byte);
int32_t	fuword32 (const uint32_t *base);
int64_t	fuword64 (const uint64_t *base);
int	suword32 (uint32_t *base, int word);
int	suword64 (uint64_t *base, uint64_t word);
uint32_t casu32(volatile uint32_t *p, uint32_t oldval, uint32_t newval);
uint64_t casu64(volatile uint64_t *p, uint64_t oldval, uint64_t newval);
uint32_t swapu32(volatile uint32_t *p, uint32_t val);
uint64_t swapu64(volatile uint64_t *p, uint64_t val);
uint32_t fuwordadd32(volatile uint32_t *p, uint32_t val);
uint64_t fuwordadd64(volatile uint64_t *p, uint64_t val);

void	DELAY(int usec);
void	DRIVERSLEEP(int usec);

void	startprofclock (struct proc *);
void	stopprofclock (struct proc *);
void	setstatclockrate (int hzrate);

/*
 * Kernel environment support functions and sundry.
 */
char	*kgetenv (const char *name);
int	ksetenv(const char *name, const char *value);
int	kunsetenv(const char *name);
void	kfreeenv(char *env);
int	ktestenv(const char *name);
int	kgetenv_int (const char *name, int *data);
int	kgetenv_string (const char *name, char *data, int size);
int	kgetenv_ulong(const char *name, unsigned long *data);
int	kgetenv_quad (const char *name, quad_t *data);
int	kgetenv_long(const char *name, long *data);
extern char *kern_envp;

#include <sys/libkern.h>

/* Initialize the world */
void	mi_startup (void);
void	nchinit (void);

/* Finalize the world. */
void	shutdown_nice (int);

/*
 * Kernel to clock driver interface.
 */
void	inittodr (time_t base);
void	resettodr (void);
void	startrtclock (void);

/* Timeouts */
typedef void timeout_t (void *);	/* timeout function type */

/* Interrupt management */

/*
 * For the alpha arch, some of these functions are static __inline, and
 * the others should be.
 */
#if defined(__x86_64__)
void		setdelayed (void);
void		setsoftcambio (void);
void		setsoftcamnet (void);
void		setsoftunused02 (void);
void		setsoftcrypto (void);
void		setsoftunused01 (void);
void		setsofttty (void);
void		setsoftvm (void);
void		setsofttq (void);
void		schedsofttty (void);
void		splz (void);
void		splz_check (void);
void		cpu_mmw_pause_int(int*, int, int, int);
void		cpu_mmw_pause_long(long*, long, int, int);
void		cpu_smp_stopped(void);
#endif /* __x86_64__ */

/*
 * Various callout lists.
 */

/* Exit callout list declarations. */
typedef void (*exitlist_fn) (struct thread *td);

int	at_exit (exitlist_fn function);
int	rm_at_exit (exitlist_fn function);

/* Fork callout list declarations. */
typedef void (*forklist_fn) (struct proc *parent, struct proc *child,
				 int flags);

int	at_fork (forklist_fn function);
int	rm_at_fork (forklist_fn function);

extern struct globaldata	*panic_cpu_gd;

/*
 * Common `proc' functions are declared here so that proc.h can be included
 * less often.
 */
int	tsleep(const volatile void *, int, const char *, int);
int	ssleep(const volatile void *, struct spinlock *, int, const char *, int)
	    __nonnull(1);
int	lksleep(const volatile void *, struct lock *, int, const char *, int)
	    __nonnull(1);
int	mtxsleep(const volatile void *, struct mtx *, int, const char *, int)
	    __nonnull(1);
int	zsleep(const volatile void *, struct lwkt_serialize *, int, const char *, int)
	    __nonnull(1);
void	tsleep_interlock(const volatile void *, int) __nonnull(1);
void	tsleep_remove (struct thread *);
int	lwkt_sleep (const char *, int);
void	tstop (void);
void	wakeup(const volatile void *chan) __nonnull(1);
void	wakeup_one(const volatile void *chan) __nonnull(1);
void	wakeup_mycpu(const volatile void *chan) __nonnull(1);
void	wakeup_mycpu_one(const volatile void *chan) __nonnull(1);
void	wakeup_oncpu(struct globaldata *gd, const volatile void *chan)
	    __nonnull(2);
void	wakeup_oncpu_one(struct globaldata *gd, const volatile void *chan)
	    __nonnull(2);
void	wakeup_domain(const volatile void *chan, int domain) __nonnull(1);
void	wakeup_domain_one(const volatile void *chan, int domain) __nonnull(1);
void	wakeup_start_delayed(void);
void	wakeup_end_delayed(void);

/*
 * Common `cdev_t' stuff are declared here to avoid #include poisoning
 */

int major(cdev_t x);
int minor(cdev_t x);
dev_t devid_from_dev(cdev_t x);
cdev_t dev_from_devid(dev_t x, int b);
int uminor(dev_t dev);
int umajor(dev_t dev);
dev_t makeudev(int x, int y);

/*
 * Unit number allocation API. (kern/subr_unit.c)
 */
struct unrhdr;
struct unrhdr *new_unrhdr(int low, int high, struct lock *lock);
void delete_unrhdr(struct unrhdr *uh);
int alloc_unr(struct unrhdr *uh);
int alloc_unrl(struct unrhdr *uh);
void free_unr(struct unrhdr *uh, u_int item);

/*
 * Population count algorithm using SWAR approach
 * - "SIMD Within A Register".
 */

static __inline uint16_t
bitcount16(uint32_t x)
{

	x = (x & 0x5555) + ((x & 0xaaaa) >> 1);
	x = (x & 0x3333) + ((x & 0xcccc) >> 2);
	x = (x + (x >> 4)) & 0x0f0f;
	x = (x + (x >> 8)) & 0x00ff;
	return (x);
}

static __inline uint32_t
bitcount32(uint32_t x)
{

	x = (x & 0x55555555) + ((x & 0xaaaaaaaa) >> 1);
	x = (x & 0x33333333) + ((x & 0xcccccccc) >> 2);
	x = (x + (x >> 4)) & 0x0f0f0f0f;
	x = (x + (x >> 8));
	x = (x + (x >> 16)) & 0x000000ff;
	return (x);
}

static __inline uint64_t
bitcount64(uint64_t x)
{

	x = (x & 0x5555555555555555) + ((x & 0xaaaaaaaaaaaaaaaa) >> 1);
	x = (x & 0x3333333333333333) + ((x & 0xcccccccccccccccc) >> 2);
	x = (x + (x >> 4)) & 0x0f0f0f0f0f0f0f0f;
	x = (x + (x >> 8));
	x = (x + (x >> 16));
	x = (x + (x >> 32)) & 0x000000ff;
	return (x);
}

/*
 * Calculate (a * b) / d with a 128-bit intermediate computation to
 * avoid overflow.
 */
static __inline uint64_t
muldivu64(uint64_t a, uint64_t b, uint64_t d)
{
	_uint128_t t;

	t = (_uint128_t)a * b;
	if (t / d > UINT64_MAX) {
		kprintf("muldivu64: overflow %lu,%lu,%lu\n", a, b, d);
		print_backtrace(-1);
	}
	return (t / d);
}

#endif /* !_SYS_SYSTM_H_ */
