src/target/armv8_cache.c

   1 /***************************************************************************
   2  *   Copyright (C) 2016 by Matthias Welwarsky                              *
   3  *   matthias.welwarsky@sysgo.com                                          *
   4  *                                                                         *
   5  *   This program is free software; you can redistribute it and/or modify  *
   6  *   it under the terms of the GNU General Public License as published by  *
   7  *   the Free Software Foundation; either version 2 of the License, or     *
   8  *   (at your option) any later version.                                   *
   9  *                                                                         *
  10  *   This program is distributed in the hope that it will be useful,       *
  11  *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
  12  *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
  13  *   GNU General Public License for more details.                          *
  14  *                                                                         *
  15  *   You should have received a copy of the GNU General Public License     *
  16  *   along with this program.  If not, see <http://www.gnu.org/licenses/>. *
  17  ***************************************************************************/
  18
  19 #ifdef HAVE_CONFIG_H
  20 #include "config.h"
  21 #endif
  22
  23 #include "armv8_cache.h"
  24 #include "armv8_dpm.h"
  25 #include "armv8_opcodes.h"
  26 #include "smp.h"
  27
  28 /* CLIDR cache types */
  29 #define CACHE_LEVEL_HAS_UNIFIED_CACHE   0x4
  30 #define CACHE_LEVEL_HAS_D_CACHE         0x2
  31 #define CACHE_LEVEL_HAS_I_CACHE         0x1
  32
  33 static int armv8_d_cache_sanity_check(struct armv8_common *armv8)
  34 {
  35         struct armv8_cache_common *armv8_cache = &armv8->armv8_mmu.armv8_cache;
  36
  37         if (armv8_cache->d_u_cache_enabled)
  38                 return ERROR_OK;
  39
  40         return ERROR_TARGET_INVALID;
  41 }
  42
  43 static int armv8_i_cache_sanity_check(struct armv8_common *armv8)
  44 {
  45         struct armv8_cache_common *armv8_cache = &armv8->armv8_mmu.armv8_cache;
  46
  47         if (armv8_cache->i_cache_enabled)
  48                 return ERROR_OK;
  49
  50         return ERROR_TARGET_INVALID;
  51 }
  52
  53 static int armv8_cache_d_inner_flush_level(struct armv8_common *armv8, struct armv8_cachesize *size, int cl)
  54 {
  55         struct arm_dpm *dpm = armv8->arm.dpm;
  56         int retval = ERROR_OK;
  57         int32_t c_way, c_index = size->index;
  58
  59         LOG_DEBUG("cl %" PRId32, cl);
  60         do {
  61                 c_way = size->way;
  62                 do {
  63                         uint32_t value = (c_index << size->index_shift)
  64                                 | (c_way << size->way_shift) | (cl << 1);
  65                         /*
  66                          * DC CISW - Clean and invalidate data cache
  67                          * line by Set/Way.
  68                          */
  69                         retval = dpm->instr_write_data_r0(dpm,
  70                                         armv8_opcode(armv8, ARMV8_OPC_DCCISW), value);
  71                         if (retval != ERROR_OK)
  72                                 goto done;
  73                         c_way -= 1;
  74                 } while (c_way >= 0);
  75                 c_index -= 1;
  76         } while (c_index >= 0);
  77
  78  done:
  79         return retval;
  80 }
  81
  82 static int armv8_cache_d_inner_clean_inval_all(struct armv8_common *armv8)
  83 {
  84         struct armv8_cache_common *cache = &(armv8->armv8_mmu.armv8_cache);
  85         struct arm_dpm *dpm = armv8->arm.dpm;
  86         int cl;
  87         int retval;
  88
  89         retval = armv8_d_cache_sanity_check(armv8);
  90         if (retval != ERROR_OK)
  91                 return retval;
  92
  93         retval = dpm->prepare(dpm);
  94         if (retval != ERROR_OK)
  95                 goto done;
  96
  97         for (cl = 0; cl < cache->loc; cl++) {
  98                 /* skip i-only caches */
  99                 if (cache->arch[cl].ctype < CACHE_LEVEL_HAS_D_CACHE)
 100                         continue;
 101
 102                 armv8_cache_d_inner_flush_level(armv8, &cache->arch[cl].d_u_size, cl);
 103         }
 104
 105         retval = dpm->finish(dpm);
 106         return retval;
 107
 108 done:
 109         LOG_ERROR("clean invalidate failed");
 110         dpm->finish(dpm);
 111
 112         return retval;
 113 }
 114
 115 int armv8_cache_d_inner_flush_virt(struct armv8_common *armv8, target_addr_t va, size_t size)
 116 {
 117         struct arm_dpm *dpm = armv8->arm.dpm;
 118         struct armv8_cache_common *armv8_cache = &armv8->armv8_mmu.armv8_cache;
 119         uint64_t linelen = armv8_cache->dminline;
 120         target_addr_t va_line, va_end;
 121         int retval;
 122
 123         retval = armv8_d_cache_sanity_check(armv8);
 124         if (retval != ERROR_OK)
 125                 return retval;
 126
 127         retval = dpm->prepare(dpm);
 128         if (retval != ERROR_OK)
 129                 goto done;
 130
 131         va_line = va & (-linelen);
 132         va_end = va + size;
 133
 134         while (va_line < va_end) {
 135                 /* DC CIVAC */
 136                 /* Aarch32: DCCIMVAC: ARMV4_5_MCR(15, 0, 0, 7, 14, 1) */
 137                 retval = dpm->instr_write_data_r0_64(dpm,
 138                                 armv8_opcode(armv8, ARMV8_OPC_DCCIVAC), va_line);
 139                 if (retval != ERROR_OK)
 140                         goto done;
 141                 va_line += linelen;
 142         }
 143
 144         dpm->finish(dpm);
 145         return retval;
 146
 147 done:
 148         LOG_ERROR("d-cache invalidate failed");
 149         dpm->finish(dpm);
 150
 151         return retval;
 152 }
 153
 154 int armv8_cache_i_inner_inval_virt(struct armv8_common *armv8, target_addr_t va, size_t size)
 155 {
 156         struct arm_dpm *dpm = armv8->arm.dpm;
 157         struct armv8_cache_common *armv8_cache = &armv8->armv8_mmu.armv8_cache;
 158         uint64_t linelen = armv8_cache->iminline;
 159         target_addr_t va_line, va_end;
 160         int retval;
 161
 162         retval = armv8_i_cache_sanity_check(armv8);
 163         if (retval != ERROR_OK)
 164                 return retval;
 165
 166         retval = dpm->prepare(dpm);
 167         if (retval != ERROR_OK)
 168                 goto done;
 169
 170         va_line = va & (-linelen);
 171         va_end = va + size;
 172
 173         while (va_line < va_end) {
 174                 /* IC IVAU - Invalidate instruction cache by VA to PoU. */
 175                 retval = dpm->instr_write_data_r0_64(dpm,
 176                                 armv8_opcode(armv8, ARMV8_OPC_ICIVAU), va_line);
 177                 if (retval != ERROR_OK)
 178                         goto done;
 179                 va_line += linelen;
 180         }
 181
 182         dpm->finish(dpm);
 183         return retval;
 184
 185 done:
 186         LOG_ERROR("d-cache invalidate failed");
 187         dpm->finish(dpm);
 188
 189         return retval;
 190 }
 191
 192 static int armv8_handle_inner_cache_info_command(struct command_invocation *cmd,
 193         struct armv8_cache_common *armv8_cache)
 194 {
 195         int cl;
 196
 197         if (armv8_cache->info == -1) {
 198                 command_print(cmd, "cache not yet identified");
 199                 return ERROR_OK;
 200         }
 201
 202         for (cl = 0; cl < armv8_cache->loc; cl++) {
 203                 struct armv8_arch_cache *arch = &(armv8_cache->arch[cl]);
 204
 205                 if (arch->ctype & 1) {
 206                         command_print(cmd,
 207                                 "L%d I-Cache: linelen %" PRIu32
 208                                 ", associativity %" PRIu32
 209                                 ", nsets %" PRIu32
 210                                 ", cachesize %" PRIu32 " KBytes",
 211                                 cl+1,
 212                                 arch->i_size.linelen,
 213                                 arch->i_size.associativity,
 214                                 arch->i_size.nsets,
 215                                 arch->i_size.cachesize);
 216                 }
 217
 218                 if (arch->ctype >= 2) {
 219                         command_print(cmd,
 220                                 "L%d D-Cache: linelen %" PRIu32
 221                                 ", associativity %" PRIu32
 222                                 ", nsets %" PRIu32
 223                                 ", cachesize %" PRIu32 " KBytes",
 224                                 cl+1,
 225                                 arch->d_u_size.linelen,
 226                                 arch->d_u_size.associativity,
 227                                 arch->d_u_size.nsets,
 228                                 arch->d_u_size.cachesize);
 229                 }
 230         }
 231
 232         return ERROR_OK;
 233 }
 234
 235 static int _armv8_flush_all_data(struct target *target)
 236 {
 237         return armv8_cache_d_inner_clean_inval_all(target_to_armv8(target));
 238 }
 239
 240 static int  armv8_flush_all_data(struct target *target)
 241 {
 242         int retval = ERROR_FAIL;
 243         /*  check that armv8_cache is correctly identify */
 244         struct armv8_common *armv8 = target_to_armv8(target);
 245         if (armv8->armv8_mmu.armv8_cache.info == -1) {
 246                 LOG_ERROR("trying to flush un-identified cache");
 247                 return retval;
 248         }
 249
 250         if (target->smp) {
 251                 /*  look if all the other target have been flushed in order to flush level
 252                  *  2 */
 253                 struct target_list *head;
 254                 foreach_smp_target(head, target->smp_targets) {
 255                         struct target *curr = head->target;
 256                         if (curr->state == TARGET_HALTED) {
 257                                 LOG_INFO("Wait flushing data l1 on core %" PRId32, curr->coreid);
 258                                 retval = _armv8_flush_all_data(curr);
 259                         }
 260                 }
 261         } else
 262                 retval = _armv8_flush_all_data(target);
 263         return retval;
 264 }
 265
 266 static int get_cache_info(struct arm_dpm *dpm, int cl, int ct, uint32_t *cache_reg)
 267 {
 268         struct armv8_common *armv8 = dpm->arm->arch_info;
 269         int retval = ERROR_OK;
 270
 271         /*  select cache level */
 272         retval = dpm->instr_write_data_r0(dpm,
 273                         armv8_opcode(armv8, WRITE_REG_CSSELR),
 274                         (cl << 1) | (ct == 1 ? 1 : 0));
 275         if (retval != ERROR_OK)
 276                 goto done;
 277
 278         retval = dpm->instr_read_data_r0(dpm,
 279                         armv8_opcode(armv8, READ_REG_CCSIDR),
 280                         cache_reg);
 281  done:
 282         return retval;
 283 }
 284
 285 static struct armv8_cachesize decode_cache_reg(uint32_t cache_reg)
 286 {
 287         struct armv8_cachesize size;
 288         int i = 0;
 289
 290         size.linelen = 16 << (cache_reg & 0x7);
 291         size.associativity = ((cache_reg >> 3) & 0x3ff) + 1;
 292         size.nsets = ((cache_reg >> 13) & 0x7fff) + 1;
 293         size.cachesize = size.linelen * size.associativity * size.nsets / 1024;
 294
 295         /*  compute info for set way operation on cache */
 296         size.index_shift = (cache_reg & 0x7) + 4;
 297         size.index = (cache_reg >> 13) & 0x7fff;
 298         size.way = ((cache_reg >> 3) & 0x3ff);
 299
 300         while (((size.way << i) & 0x80000000) == 0)
 301                 i++;
 302         size.way_shift = i;
 303
 304         return size;
 305 }
 306
 307 int armv8_identify_cache(struct armv8_common *armv8)
 308 {
 309         /*  read cache descriptor */
 310         int retval = ERROR_FAIL;
 311         struct arm *arm = &armv8->arm;
 312         struct arm_dpm *dpm = armv8->arm.dpm;
 313         uint32_t csselr, clidr, ctr;
 314         uint32_t cache_reg;
 315         int cl, ctype;
 316         struct armv8_cache_common *cache = &(armv8->armv8_mmu.armv8_cache);
 317
 318         retval = dpm->prepare(dpm);
 319         if (retval != ERROR_OK)
 320                 goto done;
 321
 322         /* check if we're in an unprivileged mode */
 323         if (armv8_curel_from_core_mode(arm->core_mode) < SYSTEM_CUREL_EL1) {
 324                 retval = armv8_dpm_modeswitch(dpm, ARMV8_64_EL1H);
 325                 if (retval != ERROR_OK)
 326                         return retval;
 327         }
 328
 329         /* retrieve CTR */
 330         retval = dpm->instr_read_data_r0(dpm,
 331                         armv8_opcode(armv8, READ_REG_CTR), &ctr);
 332         if (retval != ERROR_OK)
 333                 goto done;
 334
 335         cache->iminline = 4UL << (ctr & 0xf);
 336         cache->dminline = 4UL << ((ctr & 0xf0000) >> 16);
 337         LOG_DEBUG("ctr %" PRIx32 " ctr.iminline %" PRIu32 " ctr.dminline %" PRIu32,
 338                  ctr, cache->iminline, cache->dminline);
 339
 340         /*  retrieve CLIDR */
 341         retval = dpm->instr_read_data_r0(dpm,
 342                         armv8_opcode(armv8, READ_REG_CLIDR), &clidr);
 343         if (retval != ERROR_OK)
 344                 goto done;
 345
 346         cache->loc = (clidr & 0x7000000) >> 24;
 347         LOG_DEBUG("Number of cache levels to PoC %" PRId32, cache->loc);
 348
 349         /*  retrieve selected cache for later restore
 350          *  MRC p15, 2,<Rd>, c0, c0, 0; Read CSSELR */
 351         retval = dpm->instr_read_data_r0(dpm,
 352                         armv8_opcode(armv8, READ_REG_CSSELR), &csselr);
 353         if (retval != ERROR_OK)
 354                 goto done;
 355
 356         /* retrieve all available inner caches */
 357         for (cl = 0; cl < cache->loc; clidr >>= 3, cl++) {
 358
 359                 /* isolate cache type at current level */
 360                 ctype = clidr & 7;
 361
 362                 /* skip reserved values */
 363                 if (ctype > CACHE_LEVEL_HAS_UNIFIED_CACHE)
 364                         continue;
 365
 366                 /* separate d or unified d/i cache at this level ? */
 367                 if (ctype & (CACHE_LEVEL_HAS_UNIFIED_CACHE | CACHE_LEVEL_HAS_D_CACHE)) {
 368                         /* retrieve d-cache info */
 369                         retval = get_cache_info(dpm, cl, 0, &cache_reg);
 370                         if (retval != ERROR_OK)
 371                                 goto done;
 372                         cache->arch[cl].d_u_size = decode_cache_reg(cache_reg);
 373
 374                         LOG_DEBUG("data/unified cache index %" PRIu32 " << %" PRIu32 ", way %" PRIu32 " << %" PRIu32,
 375                                         cache->arch[cl].d_u_size.index,
 376                                         cache->arch[cl].d_u_size.index_shift,
 377                                         cache->arch[cl].d_u_size.way,
 378                                         cache->arch[cl].d_u_size.way_shift);
 379
 380                         LOG_DEBUG("cacheline %" PRIu32 " bytes %" PRIu32 " KBytes asso %" PRIu32 " ways",
 381                                         cache->arch[cl].d_u_size.linelen,
 382                                         cache->arch[cl].d_u_size.cachesize,
 383                                         cache->arch[cl].d_u_size.associativity);
 384                 }
 385
 386                 /* separate i-cache at this level ? */
 387                 if (ctype & CACHE_LEVEL_HAS_I_CACHE) {
 388                         /* retrieve i-cache info */
 389                         retval = get_cache_info(dpm, cl, 1, &cache_reg);
 390                         if (retval != ERROR_OK)
 391                                 goto done;
 392                         cache->arch[cl].i_size = decode_cache_reg(cache_reg);
 393
 394                         LOG_DEBUG("instruction cache index %" PRIu32 " << %" PRIu32 ", way %" PRIu32 " << %" PRIu32,
 395                                         cache->arch[cl].i_size.index,
 396                                         cache->arch[cl].i_size.index_shift,
 397                                         cache->arch[cl].i_size.way,
 398                                         cache->arch[cl].i_size.way_shift);
 399
 400                         LOG_DEBUG("cacheline %" PRIu32 " bytes %" PRIu32 " KBytes asso %" PRIu32 " ways",
 401                                         cache->arch[cl].i_size.linelen,
 402                                         cache->arch[cl].i_size.cachesize,
 403                                         cache->arch[cl].i_size.associativity);
 404                 }
 405
 406                 cache->arch[cl].ctype = ctype;
 407         }
 408
 409         /*  restore selected cache  */
 410         dpm->instr_write_data_r0(dpm,
 411                         armv8_opcode(armv8, WRITE_REG_CSSELR), csselr);
 412         if (retval != ERROR_OK)
 413                 goto done;
 414
 415         armv8->armv8_mmu.armv8_cache.info = 1;
 416
 417         /*  if no l2 cache initialize l1 data cache flush function function */
 418         if (!armv8->armv8_mmu.armv8_cache.flush_all_data_cache) {
 419                 armv8->armv8_mmu.armv8_cache.display_cache_info =
 420                         armv8_handle_inner_cache_info_command;
 421                 armv8->armv8_mmu.armv8_cache.flush_all_data_cache =
 422                         armv8_flush_all_data;
 423         }
 424
 425 done:
 426         armv8_dpm_modeswitch(dpm, ARM_MODE_ANY);
 427         dpm->finish(dpm);
 428         return retval;
 429
 430 }