/*	$NetBSD: cache.c,v 1.97 2007/03/04 09:03:34 macallan Exp $ */

/*
 * Copyright (c) 1996
 *	The President and Fellows of Harvard College. All rights reserved.
 * Copyright (c) 1992, 1993
 *	The Regents of the University of California.  All rights reserved.
 *
 * This software was developed by the Computer Systems Engineering group
 * at Lawrence Berkeley Laboratory under DARPA contract BG 91-66 and
 * contributed to Berkeley.
 *
 * All advertising materials mentioning features or use of this software
 * must display the following acknowledgement:
 *	This product includes software developed by Harvard University.
 *	This product includes software developed by the University of
 *	California, Lawrence Berkeley Laboratory.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. All advertising materials mentioning features or use of this software
 *    must display the following acknowledgement:
 *	This product includes software developed by Aaron Brown and
 *	Harvard University.
 *	This product includes software developed by the University of
 *	California, Berkeley and its contributors.
 * 4. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 *	@(#)cache.c	8.2 (Berkeley) 10/30/93
 *
 */

/*
 * Cache routines.
 *
 * TODO:
 *	- rework range flush
 */

#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: cache.c,v 1.97 2007/03/04 09:03:34 macallan Exp $");

#include "opt_multiprocessor.h"
#include "opt_sparc_arch.h"

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>

#include <uvm/uvm_extern.h>

#include <machine/ctlreg.h>
#include <machine/pte.h>

#include <sparc/sparc/asm.h>
#include <sparc/sparc/cache.h>
#include <sparc/sparc/cpuvar.h>

struct evcnt vcache_flush_pg =
	EVCNT_INITIALIZER(EVCNT_TYPE_INTR,0,"vcfl","pg");
EVCNT_ATTACH_STATIC(vcache_flush_pg);
struct evcnt vcache_flush_seg =
	EVCNT_INITIALIZER(EVCNT_TYPE_INTR,0,"vcfl","seg");
EVCNT_ATTACH_STATIC(vcache_flush_seg);
struct evcnt vcache_flush_reg =
	EVCNT_INITIALIZER(EVCNT_TYPE_INTR,0,"vcfl","reg");
EVCNT_ATTACH_STATIC(vcache_flush_reg);
struct evcnt vcache_flush_ctx =
	EVCNT_INITIALIZER(EVCNT_TYPE_INTR,0,"vcfl","ctx");
EVCNT_ATTACH_STATIC(vcache_flush_ctx);
struct evcnt vcache_flush_range =
	EVCNT_INITIALIZER(EVCNT_TYPE_INTR,0,"vcfl","rng");
EVCNT_ATTACH_STATIC(vcache_flush_range);

int cache_alias_dist;		/* Cache anti-aliasing constants */
int cache_alias_bits;
u_long dvma_cachealign;

/*
 * Enable the cache.
 * We need to clear out the valid bits first.
 */
void
sun4_cache_enable(void)
{
	u_int i, lim, ls, ts;

	cache_alias_bits = CPU_ISSUN4
				? CACHE_ALIAS_BITS_SUN4
				: CACHE_ALIAS_BITS_SUN4C;
	cache_alias_dist = CPU_ISSUN4
				? CACHE_ALIAS_DIST_SUN4
				: CACHE_ALIAS_DIST_SUN4C;

	ls = CACHEINFO.c_linesize;
	ts = CACHEINFO.c_totalsize;

	for (i = AC_CACHETAGS, lim = i + ts; i < lim; i += ls)
		sta(i, ASI_CONTROL, 0);

	stba(AC_SYSENABLE, ASI_CONTROL,
	     lduba(AC_SYSENABLE, ASI_CONTROL) | SYSEN_CACHE);
	CACHEINFO.c_enabled = 1;

#ifdef notyet
	if (cpuinfo.flags & SUN4_IOCACHE) {
		stba(AC_SYSENABLE, ASI_CONTROL,
		     lduba(AC_SYSENABLE, ASI_CONTROL) | SYSEN_IOCACHE);
		printf("iocache enabled\n");
	}
#endif
}

/*
 * XXX Hammer is a bit too big, here; SUN4D systems only have Viking.
 */
#if defined(SUN4M) || defined(SUN4D)
void
ms1_cache_enable(void)
{
	u_int pcr;

	cache_alias_dist = max(
		CACHEINFO.ic_totalsize / CACHEINFO.ic_associativity,
		CACHEINFO.dc_totalsize / CACHEINFO.dc_associativity);
	cache_alias_bits = (cache_alias_dist - 1) & ~PGOFSET;

	pcr = lda(SRMMU_PCR, ASI_SRMMU);

	/* We "flash-clear" the I/D caches. */
	if ((pcr & MS1_PCR_ICE) == 0)
		sta(0, ASI_ICACHECLR, 0);
	if ((pcr & MS1_PCR_DCE) == 0)
		sta(0, ASI_DCACHECLR, 0);

	/* Turn on caches */
	sta(SRMMU_PCR, ASI_SRMMU, pcr | MS1_PCR_DCE | MS1_PCR_ICE);

	CACHEINFO.c_enabled = CACHEINFO.dc_enabled = 1;

	/*
	 * When zeroing or copying pages, there might still be entries in
	 * the cache, since we don't flush pages from the cache when
	 * unmapping them (`vactype' is VAC_NONE).  Fortunately, the
	 * MS1 cache is write-through and not write-allocate, so we can
	 * use cacheable access while not displacing cache lines.
	 */
	cpuinfo.flags |= CPUFLG_CACHE_MANDATORY;
}

void
viking_cache_enable(void)
{
	u_int pcr;

	pcr = lda(SRMMU_PCR, ASI_SRMMU);

	if ((pcr & VIKING_PCR_ICE) == 0) {
		/* I-cache not on; "flash-clear" it now. */
		sta(0x80000000, ASI_ICACHECLR, 0);	/* Unlock */
		sta(0, ASI_ICACHECLR, 0);		/* clear */
	}
	if ((pcr & VIKING_PCR_DCE) == 0) {
		/* D-cache not on: "flash-clear" it. */
		sta(0x80000000, ASI_DCACHECLR, 0);
		sta(0, ASI_DCACHECLR, 0);
	}

	/* Turn on caches via MMU */
	sta(SRMMU_PCR, ASI_SRMMU, pcr | VIKING_PCR_DCE | VIKING_PCR_ICE);

	CACHEINFO.c_enabled = CACHEINFO.dc_enabled = 1;

	/* Now turn on MultiCache if it exists */
	if (cpuinfo.mxcc && CACHEINFO.ec_totalsize > 0) {
		/* Set external cache enable bit in MXCC control register */
		stda(MXCC_CTRLREG, ASI_CONTROL,
		     ldda(MXCC_CTRLREG, ASI_CONTROL) | MXCC_CTRLREG_CE);
		cpuinfo.flags |= CPUFLG_CACHEPAGETABLES; /* Ok to cache PTEs */
		CACHEINFO.ec_enabled = 1;
	}
}

void
hypersparc_cache_enable(void)
{
	int i, ls, ts;
	u_int pcr, v;
	int alias_dist;

	/*
	 * Setup the anti-aliasing constants and DVMA alignment constraint.
	 */
	alias_dist = CACHEINFO.c_totalsize;
	if (alias_dist > cache_alias_dist) {
		cache_alias_dist = alias_dist;
		cache_alias_bits = (alias_dist - 1) & ~PGOFSET;
		dvma_cachealign = cache_alias_dist;
	}

	ls = CACHEINFO.c_linesize;
	ts = CACHEINFO.c_totalsize;
	pcr = lda(SRMMU_PCR, ASI_SRMMU);

	/* Now reset cache tag memory if cache not yet enabled */
	if ((pcr & HYPERSPARC_PCR_CE) == 0)
		for (i = 0; i < ts; i += ls)
			sta(i, ASI_DCACHETAG, 0);

	pcr &= ~(HYPERSPARC_PCR_CE | HYPERSPARC_PCR_CM);
	hypersparc_cache_flush_all();

	/* Enable write-back cache */
	pcr |= HYPERSPARC_PCR_CE;
	if (CACHEINFO.c_vactype == VAC_WRITEBACK)
		pcr |= HYPERSPARC_PCR_CM;

	sta(SRMMU_PCR, ASI_SRMMU, pcr);
	CACHEINFO.c_enabled = 1;

	/* XXX: should add support */
	if (CACHEINFO.c_hwflush)
		panic("cache_enable: can't handle 4M with hw-flush cache");

	/*
	 * Enable instruction cache and, on single-processor machines,
	 * disable `Unimplemented Flush Traps'.
	 */
	v = HYPERSPARC_ICCR_ICE | (sparc_ncpus <= 1 ? HYPERSPARC_ICCR_FTD : 0);
	wrasr(v, HYPERSPARC_ASRNUM_ICCR);
}


void
swift_cache_enable(void)
{
	int i, ls, ts;
	u_int pcr;

	cache_alias_dist = max(
		CACHEINFO.ic_totalsize / CACHEINFO.ic_associativity,
		CACHEINFO.dc_totalsize / CACHEINFO.dc_associativity);
	cache_alias_bits = (cache_alias_dist - 1) & ~PGOFSET;

	pcr = lda(SRMMU_PCR, ASI_SRMMU);

	/* Now reset cache tag memory if cache not yet enabled */
	ls = CACHEINFO.ic_linesize;
	ts = CACHEINFO.ic_totalsize;
	if ((pcr & SWIFT_PCR_ICE) == 0)
		for (i = 0; i < ts; i += ls)
			sta(i, ASI_ICACHETAG, 0);

	ls = CACHEINFO.dc_linesize;
	ts = CACHEINFO.dc_totalsize;
	if ((pcr & SWIFT_PCR_DCE) == 0)
		for (i = 0; i < ts; i += ls)
			sta(i, ASI_DCACHETAG, 0);

	pcr |= (SWIFT_PCR_ICE | SWIFT_PCR_DCE);
	sta(SRMMU_PCR, ASI_SRMMU, pcr);
	CACHEINFO.c_enabled = 1;
}

void
cypress_cache_enable(void)
{
	int i, ls, ts;
	u_int pcr;
	int alias_dist;

	alias_dist = CACHEINFO.c_totalsize;
	if (alias_dist > cache_alias_dist) {
		cache_alias_dist = alias_dist;
		cache_alias_bits = (alias_dist - 1) & ~PGOFSET;
		dvma_cachealign = alias_dist;
	}

	pcr = lda(SRMMU_PCR, ASI_SRMMU);
	pcr &= ~CYPRESS_PCR_CM;

	/* Now reset cache tag memory if cache not yet enabled */
	ls = CACHEINFO.c_linesize;
	ts = CACHEINFO.c_totalsize;
	if ((pcr & CYPRESS_PCR_CE) == 0)
		for (i = 0; i < ts; i += ls)
			sta(i, ASI_DCACHETAG, 0);

	pcr |= CYPRESS_PCR_CE;
	/* If put in write-back mode, turn it on */
	if (CACHEINFO.c_vactype == VAC_WRITEBACK)
		pcr |= CYPRESS_PCR_CM;
	sta(SRMMU_PCR, ASI_SRMMU, pcr);
	CACHEINFO.c_enabled = 1;
}

void
turbosparc_cache_enable(void)
{
	int i, ls, ts;
	u_int pcr, pcf;
	/* External cache sizes in KB; see Turbo sparc manual */
	static const int ts_ecache_table[8] = {0,256,512,1024,512,1024,1024,0};

	cache_alias_dist = max(
		CACHEINFO.ic_totalsize / CACHEINFO.ic_associativity,
		CACHEINFO.dc_totalsize / CACHEINFO.dc_associativity);
	cache_alias_bits = (cache_alias_dist - 1) & ~PGOFSET;

	pcr = lda(SRMMU_PCR, ASI_SRMMU);

	/* Now reset cache tag memory if cache not yet enabled */
	ls = CACHEINFO.ic_linesize;
	ts = CACHEINFO.ic_totalsize;
	if ((pcr & TURBOSPARC_PCR_ICE) == 0)
		for (i = 0; i < ts; i += ls)
			sta(i, ASI_ICACHETAG, 0);

	ls = CACHEINFO.dc_linesize;
	ts = CACHEINFO.dc_totalsize;
	if ((pcr & TURBOSPARC_PCR_DCE) == 0)
		for (i = 0; i < ts; i += ls)
			sta(i, ASI_DCACHETAG, 0);

	pcr |= (TURBOSPARC_PCR_ICE | TURBOSPARC_PCR_DCE);
	sta(SRMMU_PCR, ASI_SRMMU, pcr);

	pcf = lda(SRMMU_PCFG, ASI_SRMMU);
	if (pcf & TURBOSPARC_PCFG_SE) {
		/*
		 * Record external cache info. The Turbosparc's second-
		 * level cache is physically addressed/tagged and is
		 * not exposed by the PROM.
		 */
		CACHEINFO.ec_totalsize = 1024 *
			ts_ecache_table[(pcf & TURBOSPARC_PCFG_SCC)];
		CACHEINFO.ec_linesize = 32;
	}
	if (pcf & TURBOSPARC_PCFG_SNP)
		printf(": DVMA coherent ");

	CACHEINFO.c_enabled = 1;
}
#endif /* SUN4M || SUN4D */


/*
 * Note: the sun4 & sun4c the cache flush functions ignore the `ctx'
 * parameter. This can be done since the pmap operations that need
 * to flush cache lines will already have switched to the proper
 * context to manipulate the MMU. Hence we can avoid the overhead
 * if saving and restoring the context here.
 */

/*
 * Flush the current context from the cache.
 *
 * This is done by writing to each cache line in the `flush context'
 * address space (or, for hardware flush, once to each page in the
 * hardware flush space, for all cache pages).
 */
void
sun4_vcache_flush_context(int ctx)
{
	char *p;
	int i, ls;

	vcache_flush_ctx.ev_count++;
	p = (char *)0;	/* addresses 0..cacheinfo.c_totalsize will do fine */
	if (CACHEINFO.c_hwflush) {
		ls = PAGE_SIZE;
		i = CACHEINFO.c_totalsize >> PGSHIFT;
		for (; --i >= 0; p += ls)
			sta(p, ASI_HWFLUSHCTX, 0);
	} else {
		ls = CACHEINFO.c_linesize;
		i = CACHEINFO.c_nlines;
		for (; --i >= 0; p += ls)
			sta(p, ASI_FLUSHCTX, 0);
	}
}

/*
 * Flush the given virtual region from the cache.
 *
 * This is also done by writing to each cache line, except that
 * now the addresses must include the virtual region number, and
 * we use the `flush region' space.
 *
 * This function is only called on sun4's with 3-level MMUs; there's
 * no hw-flush space.
 */
void
sun4_vcache_flush_region(int vreg, int ctx)
{
	int i, ls;
	char *p;

	vcache_flush_reg.ev_count++;
	p = (char *)VRTOVA(vreg);	/* reg..reg+sz rather than 0..sz */
	ls = CACHEINFO.c_linesize;
	i = CACHEINFO.c_nlines;
	for (; --i >= 0; p += ls)
		sta(p, ASI_FLUSHREG, 0);
}

/*
 * Flush the given virtual segment from the cache.
 *
 * This is also done by writing to each cache line, except that
 * now the addresses must include the virtual segment number, and
 * we use the `flush segment' space.
 *
 * Again, for hardware, we just write each page (in hw-flush space).
 */
void
sun4_vcache_flush_segment(int vreg, int vseg, int ctx)
{
	int i, ls;
	char *p;

	vcache_flush_seg.ev_count++;
	p = (char *)VSTOVA(vreg, vseg);	/* seg..seg+sz rather than 0..sz */
	if (CACHEINFO.c_hwflush) {
		ls = PAGE_SIZE;
		i = CACHEINFO.c_totalsize >> PGSHIFT;
		for (; --i >= 0; p += ls)
			sta(p, ASI_HWFLUSHSEG, 0);
	} else {
		ls = CACHEINFO.c_linesize;
		i = CACHEINFO.c_nlines;
		for (; --i >= 0; p += ls)
			sta(p, ASI_FLUSHSEG, 0);
	}
}

/*
 * Flush the given virtual page from the cache.
 * (va is the actual address, and must be aligned on a page boundary.)
 * Again we write to each cache line.
 */
void
sun4_vcache_flush_page(int va, int ctx)
{
	int i, ls;
	char *p;

#ifdef DEBUG
	if (va & PGOFSET)
		panic("cache_flush_page: asked to flush misaligned va 0x%x",va);
#endif

	vcache_flush_pg.ev_count++;
	p = (char *)va;
	ls = CACHEINFO.c_linesize;
	i = PAGE_SIZE >> CACHEINFO.c_l2linesize;
	for (; --i >= 0; p += ls)
		sta(p, ASI_FLUSHPG, 0);
}

/*
 * Flush the given virtual page from the cache.
 * (va is the actual address, and must be aligned on a page boundary.)
 * This version uses hardware-assisted flush operation and just needs
 * one write into ASI_HWFLUSHPG space to flush all cache lines.
 */
void
sun4_vcache_flush_page_hw(int va, int ctx)
{
	char *p;

#ifdef DEBUG
	if (va & PGOFSET)
		panic("cache_flush_page: asked to flush misaligned va 0x%x",va);
#endif

	vcache_flush_pg.ev_count++;
	p = (char *)va;
	sta(p, ASI_HWFLUSHPG, 0);
}

/*
 * Flush a range of virtual addresses (in the current context).
 * The first byte is at (base&~PGOFSET) and the last one is just
 * before byte (base+len).
 *
 * We choose the best of (context,segment,page) here.
 */

#define CACHE_FLUSH_MAGIC	(CACHEINFO.c_totalsize / PAGE_SIZE)

void
sun4_cache_flush(void *base, u_int len)
{
	int i, ls, baseoff;
	char *p;

	if (CACHEINFO.c_vactype == VAC_NONE)
		return;

	/*
	 * Figure out how much must be flushed.
	 *
	 * If we need to do CACHE_FLUSH_MAGIC pages,  we can do a segment
	 * in the same number of loop iterations.  We can also do the whole
	 * region. If we need to do between 2 and NSEGRG, do the region.
	 * If we need to do two or more regions, just go ahead and do the
	 * whole context. This might not be ideal (e.g., fsck likes to do
	 * 65536-byte reads, which might not necessarily be aligned).
	 *
	 * We could try to be sneaky here and use the direct mapping
	 * to avoid flushing things `below' the start and `above' the
	 * ending address (rather than rounding to whole pages and
	 * segments), but I did not want to debug that now and it is
	 * not clear it would help much.
	 *
	 * (XXX the magic number 16 is now wrong, must review policy)
	 */
	baseoff = (int)base & PGOFSET;
	i = (baseoff + len + PGOFSET) >> PGSHIFT;

	vcache_flush_range.ev_count++;

	if (__predict_true(i < CACHE_FLUSH_MAGIC)) {
		/* cache_flush_page, for i pages */
		p = (char *)((int)base & ~baseoff);
		if (CACHEINFO.c_hwflush) {
			for (; --i >= 0; p += PAGE_SIZE)
				sta(p, ASI_HWFLUSHPG, 0);
		} else {
			ls = CACHEINFO.c_linesize;
			i <<= PGSHIFT - CACHEINFO.c_l2linesize;
			for (; --i >= 0; p += ls)
				sta(p, ASI_FLUSHPG, 0);
		}
		return;
	}

	baseoff = (u_int)base & SGOFSET;
	i = (baseoff + len + SGOFSET) >> SGSHIFT;
	if (__predict_true(i == 1)) {
		sun4_vcache_flush_segment(VA_VREG(base), VA_VSEG(base), 0);
		return;
	}

	if (HASSUN4_MMU3L) {
		baseoff = (u_int)base & RGOFSET;
		i = (baseoff + len + RGOFSET) >> RGSHIFT;
		if (i == 1)
			sun4_vcache_flush_region(VA_VREG(base), 0);
		else
			sun4_vcache_flush_context(0);
	} else
		sun4_vcache_flush_context(0);
}


#if defined(SUN4M) || defined(SUN4D)
#define trapoff()	do { setpsr(getpsr() & ~PSR_ET); } while(0)
#define trapon()	do { setpsr(getpsr() | PSR_ET); } while(0)
/*
 * Flush the current context from the cache.
 *
 * This is done by writing to each cache line in the `flush context'
 * address space.
 */
void
srmmu_vcache_flush_context(int ctx)
{
	int i, ls, octx;
	char *p;

	vcache_flush_ctx.ev_count++;
	p = (char *)0;	/* addresses 0..cacheinfo.c_totalsize will do fine */
	ls = CACHEINFO.c_linesize;
	i = CACHEINFO.c_nlines;
	octx = getcontext4m();
	trapoff();
	setcontext4m(ctx);
	for (; --i >= 0; p += ls)
		sta(p, ASI_IDCACHELFC, 0);
	setcontext4m(octx);
	trapon();
}

/*
 * Flush the given virtual region from the cache.
 *
 * This is also done by writing to each cache line, except that
 * now the addresses must include the virtual region number, and
 * we use the `flush region' space.
 */
void
srmmu_vcache_flush_region(int vreg, int ctx)
{
	int i, ls, octx;
	char *p;

	vcache_flush_reg.ev_count++;
	p = (char *)VRTOVA(vreg);	/* reg..reg+sz rather than 0..sz */
	ls = CACHEINFO.c_linesize;
	i = CACHEINFO.c_nlines;
	octx = getcontext4m();
	trapoff();
	setcontext4m(ctx);
	for (; --i >= 0; p += ls)
		sta(p, ASI_IDCACHELFR, 0);
	setcontext4m(octx);
	trapon();
}

/*
 * Flush the given virtual segment from the cache.
 *
 * This is also done by writing to each cache line, except that
 * now the addresses must include the virtual segment number, and
 * we use the `flush segment' space.
 *
 * Again, for hardware, we just write each page (in hw-flush space).
 */
void
srmmu_vcache_flush_segment(int vreg, int vseg, int ctx)
{
	int i, ls, octx;
	char *p;

	vcache_flush_seg.ev_count++;
	p = (char *)VSTOVA(vreg, vseg);	/* seg..seg+sz rather than 0..sz */
	ls = CACHEINFO.c_linesize;
	i = CACHEINFO.c_nlines;
	octx = getcontext4m();
	trapoff();
	setcontext4m(ctx);
	for (; --i >= 0; p += ls)
		sta(p, ASI_IDCACHELFS, 0);
	setcontext4m(octx);
	trapon();
}

/*
 * Flush the given virtual page from the cache.
 * (va is the actual address, and must be aligned on a page boundary.)
 * Again we write to each cache line.
 */
void
srmmu_vcache_flush_page(int va, int ctx)
{
	int i, ls, octx;
	char *p;

#ifdef DEBUG
	if (va & PGOFSET)
		panic("cache_flush_page: asked to flush misaligned va 0x%x",va);
#endif

	vcache_flush_pg.ev_count++;
	p = (char *)va;

	/*
	 * XXX - if called early during bootstrap, we don't have the cache
	 *	 info yet. Make up a cache line size (double-word aligned)
	 */
	if ((ls = CACHEINFO.c_linesize) == 0)
		ls = 8;
	i = PAGE_SIZE;
	octx = getcontext4m();
	trapoff();
	setcontext4m(ctx);
	for (; i > 0; p += ls, i -= ls)
		sta(p, ASI_IDCACHELFP, 0);
#if defined(MULTIPROCESSOR)
	/*
	 * The page flush operation will have caused a MMU table walk
	 * on Hypersparc because the is physically tagged. Since the pmap
	 * functions will not always cross flush it in the MP case (because
	 * may not be active on this CPU) we flush the TLB entry now.
	 */
	/*if (cpuinfo.cpu_type == CPUTYP_HS_MBUS) -- more work than it's worth */
		sta(va | ASI_SRMMUFP_L3, ASI_SRMMUFP, 0);

#endif
	setcontext4m(octx);
	trapon();
}

/*
 * Flush entire cache.
 */
void
srmmu_cache_flush_all(void)
{

	srmmu_vcache_flush_context(0);
}

void
srmmu_vcache_flush_range(int va, int len, int ctx)
{
	int i, ls, offset;
	char *p;
	int octx;

	/*
	 * XXX - if called early during bootstrap, we don't have the cache
	 *	 info yet. Make up a cache line size (double-word aligned)
	 */
	if ((ls = CACHEINFO.c_linesize) == 0)
		ls = 8;

	vcache_flush_range.ev_count++;

	/* Compute # of cache lines covered by this range */
	offset = va & (ls - 1);
	i = len + offset;
	p = (char *)(va & ~(ls - 1));

	octx = getcontext4m();
	trapoff();
	setcontext4m(ctx);
	for (; i > 0; p += ls, i -= ls)
		sta(p, ASI_IDCACHELFP, 0);

#if defined(MULTIPROCESSOR)
	if (cpuinfo.cpu_type == CPUTYP_HS_MBUS) {
		/*
		 * See hypersparc comment in srmmu_vcache_flush_page().
		 */
		offset = va & PGOFSET;
		i = (offset + len + PGOFSET) >> PGSHIFT;

		va = va & ~PGOFSET;
		for (; --i >= 0; va += PAGE_SIZE)
			sta(va | ASI_SRMMUFP_L3, ASI_SRMMUFP, 0);
	}
#endif
	setcontext4m(octx);
	trapon();
	return;
}

/*
 * Flush a range of virtual addresses (in the current context).
 *
 * We choose the best of (context,segment,page) here.
 */

void
srmmu_cache_flush(void *base, u_int len)
{
	int ctx = getcontext4m();
	int i, baseoff;


	/*
	 * Figure out the most efficient way to flush.
	 *
	 * If we need to do CACHE_FLUSH_MAGIC pages,  we can do a segment
	 * in the same number of loop iterations.  We can also do the whole
	 * region. If we need to do between 2 and NSEGRG, do the region.
	 * If we need to do two or more regions, just go ahead and do the
	 * whole context. This might not be ideal (e.g., fsck likes to do
	 * 65536-byte reads, which might not necessarily be aligned).
	 *
	 * We could try to be sneaky here and use the direct mapping
	 * to avoid flushing things `below' the start and `above' the
	 * ending address (rather than rounding to whole pages and
	 * segments), but I did not want to debug that now and it is
	 * not clear it would help much.
	 *
	 */

	if (__predict_true(len < CACHEINFO.c_totalsize)) {
#if defined(MULTIPROCESSOR)
		FXCALL3(cpuinfo.sp_vcache_flush_range,
			cpuinfo.ft_vcache_flush_range,
			(int)base, len, ctx, CPUSET_ALL);
#else
		cpuinfo.sp_vcache_flush_range((int)base, len, ctx);
#endif
		return;
	}

	baseoff = (u_int)base & SGOFSET;
	i = (baseoff + len + SGOFSET) >> SGSHIFT;
	if (__predict_true(i == 1)) {
#if defined(MULTIPROCESSOR)
		FXCALL3(cpuinfo.sp_vcache_flush_segment,
			cpuinfo.ft_vcache_flush_segment,
			VA_VREG(base), VA_VSEG(base), ctx, CPUSET_ALL);
#else
		srmmu_vcache_flush_segment(VA_VREG(base), VA_VSEG(base), ctx);
#endif
		return;
	}

	baseoff = (u_int)base & RGOFSET;
	i = (baseoff + len + RGOFSET) >> RGSHIFT;
	while (i--) {
#if defined(MULTIPROCESSOR)
		FXCALL2(cpuinfo.sp_vcache_flush_region,
		       cpuinfo.ft_vcache_flush_region,
		       VA_VREG(base), ctx, CPUSET_ALL);
#else
		srmmu_vcache_flush_region(VA_VREG(base), ctx);
#endif
		base = ((char *)base + NBPRG);
	}
}

int ms1_cacheflush_magic = 0;
#define MS1_CACHEFLUSH_MAGIC	ms1_cacheflush_magic

void
ms1_cache_flush(void *base, u_int len)
{

	/*
	 * Although physically tagged, we still need to flush the
	 * data cache after (if we have a write-through cache) or before
	 * (in case of write-back caches) DMA operations.
	 */

#if MS1_CACHEFLUSH_MAGIC
	if (len <= MS1_CACHEFLUSH_MAGIC) {
		/*
		 * If the range to be flushed is sufficiently small
		 * invalidate the covered cache lines by hand.
		 *
		 * The MicroSPARC I has a direct-mapped virtually addressed
		 * physically tagged data cache which is organised as
		 * 128 lines of 16 bytes. Virtual address bits [4-10]
		 * select the cache line. The cache tags are accessed
		 * through the standard DCACHE control space using the
		 * same address bits as those used to select the cache
		 * line in the virtual address.
		 *
		 * Note: we don't bother to compare the actual tags
		 * since that would require looking up physical addresses.
		 *
		 * The format of the tags we read from ASI_DCACHE control
		 * space is:
		 *
		 * 31     27 26            11 10         1 0
		 * +--------+----------------+------------+-+
		 * |  xxx   |    PA[26-11]   |    xxx     |V|
		 * +--------+----------------+------------+-+
		 *
		 * PA: bits 11-26 of the physical address
		 * V:  line valid bit
		 */
		int tagaddr = ((u_int)base & 0x7f0);

		len = roundup(len, 16);
		while (len != 0) {
			int tag = lda(tagaddr, ASI_DCACHETAG);
			if ((tag & 1) == 1) {
				/* Mark this cache line invalid */
				sta(tagaddr, ASI_DCACHETAG, 0);
			}
			len -= 16;
			tagaddr = (tagaddr + 16) & 0x7f0;
		}
	} else
#endif
		/* Flush entire data cache */
		sta(0, ASI_DCACHECLR, 0);
}


/*
 * Flush entire cache.
 */
void
ms1_cache_flush_all(void)
{

	/* Flash-clear both caches */
	sta(0, ASI_ICACHECLR, 0);
	sta(0, ASI_DCACHECLR, 0);
}

void
hypersparc_cache_flush_all(void)
{

	srmmu_vcache_flush_context(getcontext4m());
	/* Flush instruction cache */
	hypersparc_pure_vcache_flush();
}

void
cypress_cache_flush_all(void)
{
	extern char kernel_text[];

	char *p;
	int i, ls;

	/* Fill the cache with known read-only content */
	p = (char *)kernel_text;
	ls = CACHEINFO.c_linesize;
	i = CACHEINFO.c_nlines;
	for (; --i >= 0; p += ls)
		(*(volatile char *)p);
}


void
viking_cache_flush(void *base, u_int len)
{
}

void
viking_pcache_flush_page(paddr_t pa, int invalidate_only)
{
	int set, i;

	/*
	 * The viking's on-chip data cache is 4-way set associative,
	 * consisting of 128 sets, each holding 4 lines of 32 bytes.
	 * Note that one 4096 byte page exactly covers all 128 sets
	 * in the cache.
	 */
	if (invalidate_only) {
		u_int pa_tag = (pa >> 12);
		u_int tagaddr;
		uint64_t tag;

		/*
		 * Loop over all sets and invalidate all entries tagged
		 * with the given physical address by resetting the cache
		 * tag in ASI_DCACHETAG control space.
		 *
		 * The address format for accessing a tag is:
		 *
		 * 31   30      27   26                  11      5 4  3 2    0
		 * +------+-----+------+-------//--------+--------+----+-----+
		 * | type | xxx | line |       xxx       |  set   | xx | 0   |
		 * +------+-----+------+-------//--------+--------+----+-----+
		 *
		 * set:  the cache set tag to be read (0-127)
		 * line: the line within the set (0-3)
		 * type: 1: read set tag; 2: read physical tag
		 *
		 * The (type 2) tag read from this address is a 64-bit word
		 * formatted as follows:
		 *
		 *          5         4         4
		 * 63       6         8         0            23               0
		 * +-------+-+-------+-+-------+-+-----------+----------------+
		 * |  xxx  |V|  xxx  |D|  xxx  |S|    xxx    |    PA[35-12]   |
		 * +-------+-+-------+-+-------+-+-----------+----------------+
		 *
		 * PA: bits 12-35 of the physical address
		 * S:  line shared bit
		 * D:  line dirty bit
		 * V:  line valid bit
		 */

#define VIKING_DCACHETAG_S	0x0000010000000000ULL	/* line valid bit */
#define VIKING_DCACHETAG_D	0x0001000000000000ULL	/* line dirty bit */
#define VIKING_DCACHETAG_V	0x0100000000000000ULL	/* line shared bit */
#define VIKING_DCACHETAG_PAMASK	0x0000000000ffffffULL	/* PA tag field */

		for (set = 0; set < 128; set++) {
			/* Set set number and access type */
			tagaddr = (set << 5) | (2 << 30);

			/* Examine the tag for each line in the set */
			for (i = 0 ; i < 4; i++) {
				tag = ldda(tagaddr | (i << 26), ASI_DCACHETAG);
				/*
				 * If this is a valid tag and the PA field
				 * matches clear the tag.
				 */
				if ((tag & VIKING_DCACHETAG_PAMASK) == pa_tag &&
				    (tag & VIKING_DCACHETAG_V) != 0)
					stda(tagaddr | (i << 26),
					     ASI_DCACHETAG, 0);
			}
		}

	} else {
		extern char kernel_text[];

		/*
		 * Force the cache to validate its backing memory
		 * by displacing all cache lines with known read-only
		 * content from the start of kernel text.
		 *
		 * Note that this thrashes the entire cache. However,
		 * we currently only need to call upon this code
		 * once at boot time.
		 */
		for (set = 0; set < 128; set++) {
			int *v = (int *)(kernel_text + (set << 5));

			/*
			 * We need to read (2*associativity-1) different
			 * locations to be sure to displace the entire set.
			 */
			i = 2 * 4 - 1;
			while (i--) {
				(*(volatile int *)v);
				v += 4096;
			}
		}
	}
}
#endif /* SUN4M || SUN4D */


#if defined(MULTIPROCESSOR)
/*
 * Cache flushing on multi-processor systems involves sending
 * inter-processor messages to flush the cache on each module.
 *
 * The current context of the originating processor is passed in the
 * message. This assumes the allocation of CPU contextses is a global
 * operation (remember that the actual context tables for the CPUs
 * are distinct).
 */

void
smp_vcache_flush_page(int va, int ctx)
{

	FXCALL2(cpuinfo.sp_vcache_flush_page, cpuinfo.ft_vcache_flush_page,
		va, ctx, CPUSET_ALL);
}

void
smp_vcache_flush_segment(int vr, int vs, int ctx)
{

	FXCALL3(cpuinfo.sp_vcache_flush_segment, cpuinfo.ft_vcache_flush_segment,
		vr, vs, ctx, CPUSET_ALL);
}

void
smp_vcache_flush_region(int vr, int ctx)
{

	FXCALL2(cpuinfo.sp_vcache_flush_region, cpuinfo.ft_vcache_flush_region,
		vr, ctx, CPUSET_ALL);
}

void
smp_vcache_flush_context(int ctx)
{

	FXCALL1(cpuinfo.sp_vcache_flush_context, cpuinfo.ft_vcache_flush_context,
		ctx, CPUSET_ALL);
}
#endif /* MULTIPROCESSOR */
