623 lines
12 KiB
C
623 lines
12 KiB
C
/*
|
|
* drivers/misc/tegra-profiler/power_clk.c
|
|
*
|
|
* Copyright (c) 2013-2020, NVIDIA CORPORATION. All rights reserved.
|
|
*
|
|
* This program is free software; you can redistribute it and/or modify it
|
|
* under the terms and conditions of the GNU General Public License,
|
|
* version 2, as published by the Free Software Foundation.
|
|
*
|
|
* This program is distributed in the hope it will be useful, but WITHOUT
|
|
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
|
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
|
* more details.
|
|
*
|
|
*/
|
|
|
|
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
|
|
|
|
#include <linux/cpufreq.h>
|
|
#include <linux/clk.h>
|
|
#include <linux/notifier.h>
|
|
#include <linux/workqueue.h>
|
|
#include <linux/cpu.h>
|
|
#include <linux/timer.h>
|
|
#include <linux/err.h>
|
|
#include <linux/version.h>
|
|
|
|
#include <linux/tegra_profiler.h>
|
|
|
|
#include "power_clk.h"
|
|
#include "quadd.h"
|
|
#include "hrt.h"
|
|
#include "comm.h"
|
|
#include "debug.h"
|
|
|
|
#define PCLK_MAX_VALUES 32
|
|
|
|
struct power_clk_data {
|
|
unsigned int value;
|
|
unsigned int prev;
|
|
};
|
|
|
|
#define PCLK_NB_GPU 0
|
|
#define PCLK_NB_EMC 0
|
|
|
|
enum {
|
|
PCLK_NB_CPU_FREQ,
|
|
PCLK_NB_CPU_HOTPLUG,
|
|
PCLK_NB_CPU_MAX,
|
|
};
|
|
|
|
#define PCLK_NB_MAX PCLK_NB_CPU_MAX
|
|
|
|
struct power_clk_source {
|
|
int type;
|
|
|
|
struct clk *clkp;
|
|
struct notifier_block nb[PCLK_NB_MAX];
|
|
|
|
int cpu;
|
|
int nr;
|
|
struct power_clk_data data[PCLK_MAX_VALUES];
|
|
|
|
atomic_t active;
|
|
struct mutex lock;
|
|
};
|
|
|
|
struct power_clk_context_s {
|
|
struct power_clk_source cpu;
|
|
struct power_clk_source gpu;
|
|
struct power_clk_source emc;
|
|
|
|
struct timer_list timer;
|
|
unsigned int period;
|
|
|
|
unsigned int is_cpufreq : 1;
|
|
|
|
struct quadd_ctx *quadd_ctx;
|
|
};
|
|
|
|
static struct power_clk_context_s power_ctx;
|
|
|
|
static void make_sample(struct power_clk_source *s)
|
|
{
|
|
int i;
|
|
u32 values[PCLK_MAX_VALUES];
|
|
struct quadd_iovec vec;
|
|
|
|
struct quadd_record_data record;
|
|
struct quadd_power_rate_data *p = &record.power_rate;
|
|
|
|
record.record_type = QUADD_RECORD_TYPE_POWER_RATE;
|
|
|
|
p->type = (u8)s->type;
|
|
p->time = quadd_get_time();
|
|
p->cpu_id = (u32)s->cpu;
|
|
p->flags = 0;
|
|
|
|
if (s->type == QUADD_POWER_CLK_CPU) {
|
|
p->nr_values = 1;
|
|
values[0] = s->data[s->cpu].value;
|
|
} else {
|
|
p->nr_values = (u16)s->nr;
|
|
for (i = 0; i < s->nr; i++)
|
|
values[i] = s->data[i].value;
|
|
}
|
|
|
|
vec.base = values;
|
|
vec.len = p->nr_values * sizeof(values[0]);
|
|
|
|
quadd_put_sample(&record, &vec, 1);
|
|
}
|
|
|
|
#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0))
|
|
static void
|
|
make_sample_hotplug(int cpu, int is_online)
|
|
{
|
|
struct quadd_record_data record;
|
|
struct quadd_hotplug_data *s = &record.hotplug;
|
|
|
|
record.record_type = QUADD_RECORD_TYPE_HOTPLUG;
|
|
|
|
s->cpu = cpu;
|
|
s->is_online = is_online ? 1 : 0;
|
|
s->time = quadd_get_time();
|
|
s->reserved = 0;
|
|
|
|
quadd_put_sample(&record, NULL, 0);
|
|
}
|
|
#endif
|
|
|
|
static inline int
|
|
is_data_changed(struct power_clk_source *s)
|
|
{
|
|
int i, cpu;
|
|
|
|
if (s->type == QUADD_POWER_CLK_CPU) {
|
|
cpu = s->cpu;
|
|
return (s->data[cpu].value != s->data[cpu].prev);
|
|
}
|
|
|
|
for (i = 0; i < s->nr; i++) {
|
|
if (s->data[i].value != s->data[i].prev)
|
|
return 1;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
static inline void
|
|
update_data(struct power_clk_source *s)
|
|
{
|
|
int i, cpu;
|
|
|
|
if (s->type == QUADD_POWER_CLK_CPU) {
|
|
cpu = s->cpu;
|
|
s->data[cpu].prev = s->data[cpu].value;
|
|
return;
|
|
}
|
|
|
|
for (i = 0; i < s->nr; i++)
|
|
s->data[i].prev = s->data[i].value;
|
|
}
|
|
|
|
static void check_source(struct power_clk_source *s)
|
|
{
|
|
if (is_data_changed(s)) {
|
|
update_data(s);
|
|
make_sample(s);
|
|
}
|
|
}
|
|
|
|
static void
|
|
read_source(struct power_clk_source *s, int cpu)
|
|
{
|
|
unsigned int value;
|
|
|
|
switch (s->type) {
|
|
case QUADD_POWER_CLK_CPU:
|
|
/* update cpu frequency */
|
|
if (cpu < 0 || cpu >= max_t(int, s->nr, nr_cpu_ids)) {
|
|
pr_err_once("error: cpu id: %d\n", cpu);
|
|
break;
|
|
}
|
|
|
|
value = cpufreq_get(cpu);
|
|
|
|
if (!mutex_trylock(&s->lock))
|
|
break;
|
|
|
|
s->cpu = cpu;
|
|
s->data[cpu].value = value;
|
|
pr_debug("PCLK_CPU(%d), value: %u\n", cpu, s->data[cpu].value);
|
|
check_source(s);
|
|
|
|
mutex_unlock(&s->lock);
|
|
break;
|
|
|
|
case QUADD_POWER_CLK_GPU:
|
|
/* update gpu frequency */
|
|
if (!mutex_trylock(&s->lock))
|
|
break;
|
|
|
|
if (s->clkp)
|
|
s->data[0].value =
|
|
(unsigned int)(clk_get_rate(s->clkp) / 1000);
|
|
pr_debug("PCLK_GPU, value: %u\n", s->data[0].value);
|
|
s->cpu = cpu;
|
|
check_source(s);
|
|
|
|
mutex_unlock(&s->lock);
|
|
break;
|
|
|
|
case QUADD_POWER_CLK_EMC:
|
|
/* update emc frequency */
|
|
if (!mutex_trylock(&s->lock))
|
|
break;
|
|
|
|
if (s->clkp)
|
|
s->data[0].value =
|
|
(unsigned int)(clk_get_rate(s->clkp) / 1000);
|
|
pr_debug("PCLK_EMC, value: %u\n", s->data[0].value);
|
|
s->cpu = cpu;
|
|
check_source(s);
|
|
|
|
mutex_unlock(&s->lock);
|
|
break;
|
|
|
|
default:
|
|
pr_err_once("error: invalid power_clk type\n");
|
|
break;
|
|
}
|
|
}
|
|
|
|
static int
|
|
gpu_notifier_call(struct notifier_block *nb,
|
|
unsigned long action, void *data)
|
|
{
|
|
read_source(&power_ctx.gpu, -1);
|
|
return NOTIFY_DONE;
|
|
}
|
|
|
|
static int
|
|
emc_notifier_call(struct notifier_block *nb,
|
|
unsigned long action, void *data)
|
|
{
|
|
read_source(&power_ctx.emc, -1);
|
|
return NOTIFY_DONE;
|
|
}
|
|
|
|
static void
|
|
read_cpufreq(struct power_clk_source *s, struct cpufreq_freqs *freq, int cpu)
|
|
{
|
|
int cpufreq = freq->new;
|
|
|
|
pr_debug("cpu: %d, cpufreq: %d\n", cpu, cpufreq);
|
|
|
|
if (cpu >= s->nr) {
|
|
pr_err_once("error: cpu id: %d\n", cpu);
|
|
return;
|
|
}
|
|
|
|
s->cpu = cpu;
|
|
s->data[cpu].value = cpufreq;
|
|
|
|
pr_debug("[%d] cpufreq: %u --> %u\n",
|
|
cpu, freq->old, cpufreq);
|
|
|
|
check_source(s);
|
|
}
|
|
|
|
static int
|
|
cpufreq_notifier_call(struct notifier_block *nb,
|
|
unsigned long action, void *data)
|
|
{
|
|
int cpu;
|
|
struct cpufreq_freqs *freq;
|
|
struct power_clk_source *s = &power_ctx.cpu;
|
|
|
|
if (!atomic_read(&s->active))
|
|
return 0;
|
|
|
|
pr_debug("action: %lu\n", action);
|
|
|
|
if (action == CPUFREQ_POSTCHANGE) {
|
|
if (mutex_trylock(&s->lock)) {
|
|
freq = data;
|
|
#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 2, 0))
|
|
cpu = freq->cpu;
|
|
read_cpufreq(s, freq, cpu);
|
|
#else
|
|
for_each_cpu(cpu, freq->policy->cpus)
|
|
read_cpufreq(s, freq, cpu);
|
|
#endif
|
|
mutex_unlock(&s->lock);
|
|
}
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0))
|
|
static int
|
|
cpu_hotplug_notifier_call(struct notifier_block *nb,
|
|
unsigned long action, void *hcpu)
|
|
{
|
|
int cpu;
|
|
struct power_clk_source *s = &power_ctx.cpu;
|
|
|
|
if (!atomic_read(&s->active))
|
|
return NOTIFY_DONE;
|
|
|
|
cpu = (long)hcpu;
|
|
|
|
pr_debug("cpu: %d, action: %lu\n", cpu, action);
|
|
|
|
if (cpu >= s->nr) {
|
|
pr_err_once("error: cpu id: %d\n", cpu);
|
|
return NOTIFY_DONE;
|
|
}
|
|
|
|
switch (action) {
|
|
case CPU_ONLINE:
|
|
case CPU_ONLINE_FROZEN:
|
|
make_sample_hotplug(cpu, 1);
|
|
break;
|
|
|
|
case CPU_DEAD:
|
|
case CPU_DEAD_FROZEN:
|
|
mutex_lock(&s->lock);
|
|
if (atomic_read(&s->active))
|
|
s->data[cpu].value = 0;
|
|
mutex_unlock(&s->lock);
|
|
|
|
make_sample_hotplug(cpu, 0);
|
|
break;
|
|
|
|
default:
|
|
return NOTIFY_DONE;
|
|
}
|
|
|
|
return NOTIFY_OK;
|
|
}
|
|
#endif
|
|
|
|
static void reset_data(struct power_clk_source *s)
|
|
{
|
|
int i;
|
|
|
|
for (i = 0; i < s->nr; i++) {
|
|
s->data[i].value = 0;
|
|
s->data[i].prev = 0;
|
|
}
|
|
}
|
|
|
|
static void init_source(struct power_clk_source *s,
|
|
int nr_values,
|
|
unsigned int type)
|
|
{
|
|
s->clkp = NULL;
|
|
s->type = type;
|
|
s->nr = min_t(int, nr_values, PCLK_MAX_VALUES);
|
|
atomic_set(&s->active, 0);
|
|
mutex_init(&s->lock);
|
|
|
|
reset_data(s);
|
|
}
|
|
|
|
static void
|
|
power_clk_work_func(struct work_struct *work)
|
|
{
|
|
read_source(&power_ctx.gpu, -1);
|
|
read_source(&power_ctx.emc, -1);
|
|
}
|
|
|
|
static DECLARE_WORK(power_clk_work, power_clk_work_func);
|
|
|
|
#if (LINUX_VERSION_CODE > KERNEL_VERSION(4, 14, 0))
|
|
static void power_clk_timer(struct timer_list *t)
|
|
#else
|
|
static void power_clk_timer(unsigned long unused)
|
|
#endif
|
|
{
|
|
#if (LINUX_VERSION_CODE <= KERNEL_VERSION(4, 14, 0))
|
|
struct timer_list *t = &power_ctx.timer;
|
|
#endif
|
|
|
|
schedule_work(&power_clk_work);
|
|
mod_timer(t, jiffies + msecs_to_jiffies(power_ctx.period));
|
|
}
|
|
|
|
static void
|
|
read_all_sources_work_func(struct work_struct *work)
|
|
{
|
|
int cpu_id;
|
|
struct power_clk_source *s = &power_ctx.cpu;
|
|
|
|
if (power_ctx.is_cpufreq) {
|
|
for_each_possible_cpu(cpu_id)
|
|
read_source(s, cpu_id);
|
|
}
|
|
|
|
read_source(&power_ctx.gpu, -1);
|
|
read_source(&power_ctx.emc, -1);
|
|
}
|
|
|
|
static DECLARE_WORK(read_all_sources_work, read_all_sources_work_func);
|
|
|
|
static int
|
|
enable_clock(struct power_clk_source *s, struct notifier_block *nb,
|
|
const char *dev_id, const char *con_id)
|
|
{
|
|
int ret;
|
|
|
|
mutex_lock(&s->lock);
|
|
|
|
s->clkp = clk_get_sys(dev_id, con_id);
|
|
if (IS_ERR_OR_NULL(s->clkp)) {
|
|
pr_warn("warning: could not setup clock: \"%s:%s\"\n",
|
|
dev_id ? dev_id : "", con_id ? con_id : "");
|
|
ret = -ENOENT;
|
|
goto errout;
|
|
}
|
|
|
|
ret = clk_prepare_enable(s->clkp);
|
|
if (ret) {
|
|
pr_warn("warning: could not enable gpu clock\n");
|
|
goto errout_free_clk;
|
|
}
|
|
|
|
#ifdef CONFIG_COMMON_CLK
|
|
ret = clk_notifier_register(s->clkp, nb);
|
|
if (ret) {
|
|
pr_warn("warning: could not register clock: \"%s:%s\"\n",
|
|
dev_id ? dev_id : "", con_id ? con_id : "");
|
|
goto errout_disable_clk;
|
|
}
|
|
#endif
|
|
|
|
reset_data(s);
|
|
atomic_set(&s->active, 1);
|
|
|
|
mutex_unlock(&s->lock);
|
|
|
|
return 0;
|
|
|
|
#ifdef CONFIG_COMMON_CLK
|
|
errout_disable_clk:
|
|
clk_disable_unprepare(s->clkp);
|
|
#endif
|
|
|
|
errout_free_clk:
|
|
clk_put(s->clkp);
|
|
|
|
errout:
|
|
s->clkp = NULL;
|
|
atomic_set(&s->active, 0);
|
|
|
|
mutex_unlock(&s->lock);
|
|
|
|
return ret;
|
|
}
|
|
|
|
static void
|
|
disable_clock(struct power_clk_source *s, struct notifier_block *nb)
|
|
{
|
|
mutex_lock(&s->lock);
|
|
|
|
if (atomic_cmpxchg(&s->active, 1, 0)) {
|
|
if (s->clkp) {
|
|
#ifdef CONFIG_COMMON_CLK
|
|
clk_notifier_unregister(s->clkp, nb);
|
|
#endif
|
|
clk_disable_unprepare(s->clkp);
|
|
clk_put(s->clkp);
|
|
|
|
s->clkp = NULL;
|
|
}
|
|
}
|
|
|
|
mutex_unlock(&s->lock);
|
|
}
|
|
|
|
int quadd_power_clk_start(void)
|
|
{
|
|
struct power_clk_source *s;
|
|
struct timer_list *timer = &power_ctx.timer;
|
|
struct quadd_parameters *param = &power_ctx.quadd_ctx->param;
|
|
|
|
if (param->power_rate_freq == 0) {
|
|
pr_info("power_clk is not started\n");
|
|
return 0;
|
|
}
|
|
|
|
#ifdef CONFIG_COMMON_CLK
|
|
power_ctx.period = 0;
|
|
#else
|
|
power_ctx.period = MSEC_PER_SEC / param->power_rate_freq;
|
|
pr_info("pclk: use timer, freq: %u\n", param->power_rate_freq);
|
|
#endif
|
|
|
|
pr_info("pclk: start, cpufreq: %s\n",
|
|
power_ctx.is_cpufreq ? "yes" : "no");
|
|
|
|
/* setup gpu frequency */
|
|
s = &power_ctx.gpu;
|
|
enable_clock(s, &s->nb[PCLK_NB_GPU], "3d", NULL);
|
|
|
|
/* setup emc frequency */
|
|
s = &power_ctx.emc;
|
|
enable_clock(s, &s->nb[PCLK_NB_EMC], "cpu", "emc");
|
|
|
|
/* setup cpu frequency notifier */
|
|
if (power_ctx.is_cpufreq) {
|
|
s = &power_ctx.cpu;
|
|
mutex_lock(&s->lock);
|
|
reset_data(s);
|
|
atomic_set(&s->active, 1);
|
|
mutex_unlock(&s->lock);
|
|
}
|
|
|
|
if (power_ctx.period > 0) {
|
|
#if (LINUX_VERSION_CODE > KERNEL_VERSION(4, 14, 0))
|
|
timer_setup(timer, power_clk_timer, 0);
|
|
#else
|
|
setup_timer(timer, power_clk_timer, 0);
|
|
#endif
|
|
mod_timer(timer, jiffies + msecs_to_jiffies(power_ctx.period));
|
|
}
|
|
|
|
schedule_work(&read_all_sources_work);
|
|
|
|
return 0;
|
|
}
|
|
|
|
void quadd_power_clk_stop(void)
|
|
{
|
|
struct power_clk_source *s;
|
|
struct quadd_parameters *param = &power_ctx.quadd_ctx->param;
|
|
|
|
if (param->power_rate_freq == 0)
|
|
return;
|
|
|
|
if (power_ctx.period > 0)
|
|
del_timer_sync(&power_ctx.timer);
|
|
|
|
s = &power_ctx.gpu;
|
|
disable_clock(s, &s->nb[PCLK_NB_GPU]);
|
|
|
|
s = &power_ctx.emc;
|
|
disable_clock(s, &s->nb[PCLK_NB_EMC]);
|
|
|
|
if (power_ctx.is_cpufreq) {
|
|
s = &power_ctx.cpu;
|
|
mutex_lock(&s->lock);
|
|
atomic_set(&s->active, 0);
|
|
s->clkp = NULL;
|
|
mutex_unlock(&s->lock);
|
|
}
|
|
|
|
pr_info("pclk: stop\n");
|
|
}
|
|
|
|
int quadd_power_clk_init(struct quadd_ctx *quadd_ctx)
|
|
{
|
|
int __maybe_unused ret;
|
|
struct power_clk_source *s;
|
|
|
|
s = &power_ctx.gpu;
|
|
s->nb[PCLK_NB_GPU].notifier_call = gpu_notifier_call;
|
|
init_source(s, 1, QUADD_POWER_CLK_GPU);
|
|
|
|
s = &power_ctx.emc;
|
|
s->nb[PCLK_NB_EMC].notifier_call = emc_notifier_call;
|
|
init_source(s, 1, QUADD_POWER_CLK_EMC);
|
|
|
|
s = &power_ctx.cpu;
|
|
s->nb[PCLK_NB_CPU_FREQ].notifier_call = cpufreq_notifier_call;
|
|
#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0))
|
|
s->nb[PCLK_NB_CPU_HOTPLUG].notifier_call = cpu_hotplug_notifier_call;
|
|
#endif
|
|
init_source(s, nr_cpu_ids, QUADD_POWER_CLK_CPU);
|
|
|
|
power_ctx.quadd_ctx = quadd_ctx;
|
|
|
|
#ifdef CONFIG_CPU_FREQ
|
|
ret = cpufreq_register_notifier(&s->nb[PCLK_NB_CPU_FREQ],
|
|
CPUFREQ_TRANSITION_NOTIFIER);
|
|
if (ret < 0) {
|
|
pr_warn("CPU freq registration failed: %d\n", ret);
|
|
power_ctx.is_cpufreq = 0;
|
|
} else {
|
|
power_ctx.is_cpufreq = 1;
|
|
}
|
|
#else
|
|
power_ctx.is_cpufreq = 0;
|
|
#endif
|
|
quadd_ctx->pclk_cpufreq = power_ctx.is_cpufreq;
|
|
|
|
#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)
|
|
register_cpu_notifier(&s->nb[PCLK_NB_CPU_HOTPLUG]);
|
|
#endif
|
|
|
|
return 0;
|
|
}
|
|
|
|
void quadd_power_clk_deinit(void)
|
|
{
|
|
struct power_clk_source *s __maybe_unused = &power_ctx.cpu;
|
|
|
|
quadd_power_clk_stop();
|
|
|
|
#ifdef CONFIG_CPU_FREQ
|
|
if (power_ctx.is_cpufreq)
|
|
cpufreq_unregister_notifier(&s->nb[PCLK_NB_CPU_FREQ],
|
|
CPUFREQ_TRANSITION_NOTIFIER);
|
|
#endif
|
|
|
|
#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)
|
|
unregister_cpu_notifier(&s->nb[PCLK_NB_CPU_HOTPLUG]);
|
|
#endif
|
|
}
|