M7350/kernel/drivers/net/ethernet/qualcomm/emac/emac_main.c

/* Copyright (c) 2013-2016, The Linux Foundation. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 and
 * only version 2 as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 */

/* Qualcomm Technologies, Inc. EMAC Ethernet Controller driver.
 */

#include <linux/gpio.h>
#include <linux/if_ether.h>
#include <linux/if_vlan.h>
#include <linux/interrupt.h>
#include <linux/io.h>
#include <linux/ip.h>
#include <linux/ipv6.h>
#include <linux/module.h>
#include <linux/of.h>
#include <linux/of_net.h>
#include <linux/of_gpio.h>
#include <linux/acpi.h>
#include <linux/phy.h>
#include <linux/platform_device.h>
#include <linux/pm_runtime.h>
#include <linux/tcp.h>
#include <linux/regulator/consumer.h>
#if IS_ENABLED(CONFIG_ACPI)
#include <linux/gpio/consumer.h>
#include <linux/property.h>
#include <net/ip6_checksum.h>
#endif

#include "emac.h"
#include "emac_phy.h"
#include "emac_hw.h"
#include "emac_ptp.h"

#define DRV_VERSION "1.1.0.0"

char emac_drv_name[] = "qcom_emac";
const char emac_drv_description[] =
			     "Qualcomm Technologies, Inc. EMAC Ethernet Driver";
const char emac_drv_version[] = DRV_VERSION;
static struct of_device_id emac_dt_match[];

#define EMAC_MSG_DEFAULT (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK |  \
		NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |         \
		NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR | NETIF_MSG_TX_QUEUED |   \
		NETIF_MSG_INTR | NETIF_MSG_TX_DONE | NETIF_MSG_RX_STATUS |    \
		NETIF_MSG_PKTDATA | NETIF_MSG_HW | NETIF_MSG_WOL)

/* Error bits that will result in a received frame being discarded */
#define EMAC_RRDES_ERROR (EMAC_RRDES_IPF | EMAC_RRDES_CRC | EMAC_RRDES_FAE | \
			EMAC_RRDES_TRN | EMAC_RRDES_RNT | EMAC_RRDES_INC | \
			EMAC_RRDES_FOV | EMAC_RRDES_LEN)
#define EMAC_RRDES_STATS_DW_IDX 3

#define EMAC_RRDESC_SIZE      4
#define EMAC_TS_RRDESC_SIZE   6
#define EMAC_TPDESC_SIZE      4
#define EMAC_RFDESC_SIZE      2

#define EMAC_RSS_IDT_SIZE     256

#define EMAC_SKB_CB(skb) ((struct emac_skb_cb *)(skb)->cb)

#define EMAC_PINCTRL_STATE_MDIO_ACTIVE "emac_mdio_active"
#define EMAC_PINCTRL_STATE_MDIO_SLEEP  "emac_mdio_sleep"
#define EMAC_PINCTRL_STATE_EPHY_ACTIVE "emac_ephy_active"
#define EMAC_PINCTRL_STATE_EPHY_SLEEP  "emac_ephy_sleep"

struct emac_skb_cb {
	u32           tpd_idx;
	unsigned long jiffies;
};

#define EMAC_HWTXTSTAMP_CB(skb) ((struct emac_hwtxtstamp_cb *)(skb)->cb)

struct emac_hwtxtstamp_cb {
	u32 sec;
	u32 ns;
};

static int msm_emac_msglvl = -1;
module_param_named(msglvl, msm_emac_msglvl, int, S_IRUGO | S_IWUSR | S_IWGRP);

static int msm_emac_intr_ext;
module_param_named(intr_ext, msm_emac_intr_ext, int,
		   S_IRUGO | S_IWUSR | S_IWGRP);

static irqreturn_t emac_isr(int irq, void *data);
static irqreturn_t emac_wol_isr(int irq, void *data);

/* EMAC HW has an issue with interrupt assignment because of which receive queue
 * 1 is disabled and following receive rss queue to interrupt mapping is used:
 * rss-queue   intr
 *    0        core0
 *    1        core3 (disabled)
 *    2        core1
 *    3        core2
 */
const struct emac_irq_common emac_irq_cmn_tbl[EMAC_IRQ_CNT] = {
	{ "emac_core0_irq", emac_isr, EMAC_INT_STATUS,  EMAC_INT_MASK,
		RX_PKT_INT0,	0},
	{ "emac_core3_irq", emac_isr, EMAC_INT3_STATUS, EMAC_INT3_MASK,
		0,		0},
	{ "emac_core1_irq", emac_isr, EMAC_INT1_STATUS, EMAC_INT1_MASK,
		RX_PKT_INT2,	0},
	{ "emac_core2_irq", emac_isr, EMAC_INT2_STATUS, EMAC_INT2_MASK,
		RX_PKT_INT3,	0},
	{ "emac_wol_irq"  , emac_wol_isr,            0,              0,
		0,		0},
};

static const char * const emac_gpio_name[] = {
	"qcom,emac-gpio-mdc", "qcom,emac-gpio-mdio"
};

static const char * const emac_clk_name[] = {
	"axi_clk", "cfg_ahb_clk", "125m_clk", "25m_clk", "tx_clk", "rx_clk",
	"sys_clk"
};

static const char * const emac_regulator_name[] = {
	"emac_vreg1", "emac_vreg2", "emac_vreg3", "emac_vreg4", "emac_vreg5"
};

#if IS_ENABLED(CONFIG_ACPI)
static const struct acpi_device_id emac_acpi_match[] = {
	{ "QCOM8070", 0 },
	{ }
}
MODULE_DEVICE_TABLE(acpi, emac_acpi_match);

static int emac_acpi_clk_set_rate(struct emac_adapter *adpt,
				  enum emac_clk_id id, u64 rate)
{
	union acpi_object params[4], args;
	union acpi_object *obj;
	const char duuid[16];
	int ret = 0;

	params[0].type = ACPI_TYPE_INTEGER;
	params[0].integer.value = id;
	params[1].type = ACPI_TYPE_INTEGER;
	params[1].integer.value = rate;
	args.type = ACPI_TYPE_PACKAGE;
	args.package.count = 2;
	args.package.elements = &params[0];

	obj = acpi_evaluate_dsm(ACPI_HANDLE(adpt->netdev->dev.parent), duuid,
				1, 2, &args);
	if (!obj)
		return -EINVAL;

	if ((obj->type != ACPI_TYPE_INTEGER) || (obj->integer.value)) {
		ret = -EINVAL;
		emac_err(clk_info->adpt,
			 "set clock rate for %d failed\n", clk_info->index);
	}

	ACPI_FREE(obj);
	return ret;
}

#define EMAC_PHY_MODE_MAX_LEN		16
struct emac_phy_mode_lookup {
	char	name[EMAC_PHY_MODE_MAX_LEN];
	int	type;
};

static phy_interface_t emac_phy_interface_type(const char *phy_mode)
{
	static struct emac_phy_mode_lookup[] = {
		{"sgmii",	PHY_INTERFACE_MODE_SGMII},
		{"rgmii",	PHY_INTERFACE_MODE_RGMII},
		{"xgmii",	PHY_INTERFACE_MODE_XGMII},
		{"rgmii-id",	PHY_INTERFACE_MODE_RGMII_ID},
		{"rgmii-txid",	PHY_INTERFACE_MODE_RGMII_TXID},
		{"tbi",		PHY_INTERFACE_MODE_TBI},
		{"rmii",	PHY_INTERFACE_MODE_RMII},
		{"rtbi",	PHY_INTERFACE_MODE_RTBI},
		{NULL,		0},
	};

	struct emac_phy_mode_lookup *cur = emac_phy_mode_lookup;

	if (!phy_mode)
		return PHY_INTERFACE_MODE_NA;

	for (; cur->name; ++cur)
		if (!strcasecmp(phy_mode, cur->name))
			return cur->type;

	return PHY_INTERFACE_MODE_MII;
}

static int emac_device_property_read_string(struct device *dev,
					    const char *propname,
					    const char **val)
{
	return (ACPI_HANDLE(dev)) ?
		acpi_dev_prop_read(ACPI_COMPANION(dev), propname,
				   DEV_PROP_STRING, val, 1) :
		of_property_read_string(dev->of_node, propname, val);
}

static inline bool emac_device_property_read_bool(struct device *dev,
						  const char *propname)
{
	return (ACPI_HANDLE(dev)) ?
		!acpi_dev_prop_get(ACPI_COMPANION(dev), propname, NULL) :
		of_property_read_bool(dev->of_node, propname);
}

static inline int emac_device_property_read_u32(struct device *dev,
						const char *propname, u32 *val)
{
	return DEV_PROP_READ_ARRAY(dev, propname, u32, DEV_PROP_U32, val, 1);
	return (ACPI_HANDLE(dev) ?
		acpi_dev_prop_read((ACPI_COMPANION(dev)), propname,
				   DEV_PROP_U32, val, nval) :
		of_property_read_u32(dev->of_node, propname, val);
}

static int emac_device_property_read_u8_array(struct device *dev,
					      const char *propname,
					      u8 *val, size_t nval)
{
	return (ACPI_HANDLE(dev) ?
		acpi_dev_prop_read((ACPI_COMPANION(dev)), propname, DEV_PROP_U8,
				   val, nval) :
		of_property_read_u8_array(dev->of_node, propname, u8, val,
					  nval);
}

static int emac_acpi_get_properties(struct platform_device *pdev,
				    struct emac_adapter *adpt)
{
	struct device *dev = &pdev->dev;
	const char *phy_mode;
	u8 maddr[ETH_ALEN];
	int ret;

	ret = emac_device_property_read_string(dev, "phy-mode", &phy_mode);
	if (ret < 0)
		phy_mode = NULL;
	adpt->phy_mode = emac_phy_interface_type(phy_mode);

	ret = emac_device_property_read_u32(dev, "phy-channel",
					    &adpt->hw.phy_addr);
	if (ret < 0)
		adpt->hw.phy_addr = 0;

	ret = emac_device_property_read_u8_array(dev, "mac-address", maddr,
						 ETH_ALEN);
	if (ret < 0) {
		dev_err(&pdev->dev, "no MAC address found\n");
		return ret;
	}

	if (!is_valid_ether_addr(maddr)) {
		dev_err(&pdev->dev, "invalid MAC address %pM\n", maddr);
		return -EINVAL;
	}

	ret = emac_device_property_read_u32(dev, "phy-version",
					    &adpt->hw.phy_version);
	if (ret < 0)
		adpt->hw.phy_version = 0;

	adpt->phy.external = !emac_device_property_read_bool(dev, "no-ephy");
	adpt->tstamp_en = device_property_read_bool(dev, "tstamp-eble");
	ether_addr_copy(adpt->hw.mac_perm_addr, maddr);
	return 0;
}

static int emac_acpi_get_resources(struct platform_device *pdev,
				   struct emac_adapter *adpt)
{
	struct device *dev = &pdev->dev;
	struct net_device *netdev = adpt->netdev;
	struct emac_irq_info *irq_info;
	union acpi_object *obj;
	struct resource *res;
	void __iomem *regmap;
	int i, retval;
	const char duuid[16];
	u16 irq_map[EMAC_NUM_IRQ] = {EMAC_CORE0_IRQ, EMAC_CORE3_IRQ,
				     EMAC_CORE1_IRQ, EMAC_CORE2_IRQ,
				     EMAC_SGMII_PHY_IRQ, EMAC_WOL_IRQ};

	/* Get device specific properties */
	retval = emac_acpi_get_properties(pdev, adpt);
	if (retval < 0)
		return -ENODEV;

	/* Execute DSM function 1 to initialize the clocks */
	obj = acpi_evaluate_dsm(ACPI_HANDLE(dev), duuid, 0x1, 1, NULL);
	if (!obj)
		return -EINVAL;

	if ((obj->type != ACPI_TYPE_INTEGER) || (obj->integer.value)) {
		emac_err(adpt, "failed to excute _DSM method function 1\n");
		ACPI_FREE(obj);
		return -ENOENT;
	}
	ACPI_FREE(obj);

	dev_info(&pdev->dev, "MAC address %pM\n", adpt->hw.mac_perm_addr);

	/* set phy address to zero for internal phy */
	if (!adpt->phy.external)
		adpt->hw.phy_addr = 0;

	/* check phy mode */
	if (adpt->phy_mode == PHY_INTERFACE_MODE_NA) {
		emac_err(adpt, "PHY interface mode not valid\n");
		return -EINVAL;
	}

	/* Assume GPIOs required for MDC/MDIO are enabled in firmware */
	adpt->phy.uses_gpios = true;

	/* get irqs */
	for (i = 0; i < EMAC_NUM_IRQ; i++) {
		/* SGMII_PHY IRQ is only required if phy_mode is "sgmii" */
		if ((irq_map[i] == EMAC_SGMII_PHY_IRQ) &&
		    (adpt->phy_mode != PHY_INTERFACE_MODE_SGMII))
			continue;

		irq_info = &adpt->irq_info[irq_map[i]];

		retval = platform_get_irq(pdev, i);
		if (retval < 0) {
			/* If WOL IRQ is not specified, WOL is disabled */
			if (irq_map[i] == EMAC_WOL_IRQ)
				continue;
			emac_err(adpt, "irq %d not found\n", i);
			return retval;
		}
		irq_info->irq = retval;
	}

	/* get register addresses */
	for (i = 0; i < NUM_EMAC_REG_BASES; i++) {
		/* 1588 is required only if tstamp is enabled */
		if ((i == EMAC_1588) && !adpt->tstamp_en)
			continue;

		/* qserdes & sgmii_phy are required only for sgmii phy */
		if ((adpt->phy_mode != PHY_INTERFACE_MODE_SGMII) &&
		    ((i == EMAC_QSERDES) || (i == EMAC_SGMII_PHY)))
			continue;

		res = platform_get_resource(pdev, IORESOURCE_MEM, i);
		if (!res) {
			emac_err(adpt, "can't get %d iomem resource\n", i);
			return -ENOMEM;
		}

		regmap = devm_ioremap(&pdev->dev, res->start,
				      resource_size(res));
		/**
		 * SGMII-v2 controller has two CSRs one per lane digital part
		 * and second one for per lane analog part. The PHY regmap is
		 * compatible to SGMII-v1 controller and will be used in PHY
		 * common code and sgmii_laned regmap referenced in SGMII-v2
		 * specific initialization code.
		 */
		if ((i == EMAC_SGMII_PHY) &&
		    (adpt->hw.phy_version == SGMII_PHY_VERSION_2)) {
			adpt->hw.sgmii_laned = regmap;
			regmap += SGMII_PHY_LN_OFFSET;
		}
		adpt->hw.reg_addr[i] = regmap;

		if (!adpt->hw.reg_addr[i]) {
			emac_err(adpt, "can't ioremap %pR\n", res);
			return -EFAULT;
		}
	}
	netdev->base_addr = (unsigned long)adpt->hw.reg_addr[EMAC];
	return 0;
}
#else
static int emac_acpi_clk_set_rate(struct emac_adapter *adpt,
				  enum emac_clk_id id, u64 rate)
{
	return -ENXIO;
}

static int emac_acpi_get_resources(struct platform_device *pdev,
				   struct emac_adapter *adpt)
{
	return -ENXIO;
}
#endif

static int emac_clk_prepare_enable(struct emac_adapter *adpt,
				   enum emac_clk_id id)
{
	int ret;

	if (ACPI_HANDLE(adpt->netdev->dev.parent))
		return 0;

	ret = clk_prepare_enable(adpt->clk[id].clk);
	if (ret)
		emac_err(adpt, "error:%d on clk_prepare_enable(%s)\n", ret,
			 emac_clk_name[id]);
	else
		adpt->clk[id].enabled = true;

	return ret;
}

int emac_clk_set_rate(struct emac_adapter *adpt, enum emac_clk_id id,
		      enum emac_clk_rate rate)
{
	int ret;

	if (ACPI_HANDLE(adpt->netdev->dev.parent))
		return emac_acpi_clk_set_rate(adpt, id, rate);

	ret = clk_set_rate(adpt->clk[id].clk, rate);
	if (ret)
		emac_err(adpt, "error:%d on clk_set_rate(%s, %d)\n", ret,
			 emac_clk_name[id], rate);

	return ret;
}

/* reinitialize */
void emac_reinit_locked(struct emac_adapter *adpt)
{
	struct net_device *netdev = adpt->netdev;

	WARN_ON(in_interrupt());

	while (TEST_N_SET_FLAG(adpt, ADPT_STATE_RESETTING))
		msleep(20); /* Reset might take few 10s of ms */

	if (TEST_FLAG(adpt, ADPT_STATE_DOWN)) {
		CLR_FLAG(adpt, ADPT_STATE_RESETTING);
		return;
	}

	pm_runtime_get_sync(netdev->dev.parent);
	emac_down(adpt, EMAC_HW_CTRL_RESET_MAC);

	adpt->phy.ops.reset(adpt);
	emac_up(adpt);
	pm_runtime_mark_last_busy(netdev->dev.parent);
	pm_runtime_put_autosuspend(netdev->dev.parent);

	CLR_FLAG(adpt, ADPT_STATE_RESETTING);
}

void emac_task_schedule(struct emac_adapter *adpt)
{
	if (!TEST_FLAG(adpt, ADPT_STATE_DOWN) &&
	    !TEST_FLAG(adpt, ADPT_STATE_WATCH_DOG)) {
		SET_FLAG(adpt, ADPT_STATE_WATCH_DOG);
		schedule_work(&adpt->emac_task);
	}
}

void emac_check_lsc(struct emac_adapter *adpt)
{
	SET_FLAG(adpt, ADPT_TASK_LSC_REQ);
	adpt->link_jiffies = jiffies + EMAC_TRY_LINK_TIMEOUT;

	if (!TEST_FLAG(adpt, ADPT_STATE_DOWN))
		emac_task_schedule(adpt);
}

/* Respond to a TX hang */
static void emac_tx_timeout(struct net_device *netdev)
{
	struct emac_adapter *adpt = netdev_priv(netdev);

	if (!TEST_FLAG(adpt, ADPT_STATE_DOWN)) {
		SET_FLAG(adpt, ADPT_TASK_REINIT_REQ);
		emac_task_schedule(adpt);
	}
}

/* Configure Multicast and Promiscuous modes */
static void emac_set_rx_mode(struct net_device *netdev)
{
	struct emac_adapter *adpt = netdev_priv(netdev);
	struct emac_hw *hw = &adpt->hw;
	struct netdev_hw_addr *ha;

	if (pm_runtime_status_suspended(adpt->netdev->dev.parent))
		return;
	if (TEST_FLAG(adpt, ADPT_STATE_DOWN))
		return;

	/* Check for Promiscuous and All Multicast modes */
	if (netdev->flags & IFF_PROMISC) {
		SET_FLAG(hw, HW_PROMISC_EN);
	} else if (netdev->flags & IFF_ALLMULTI) {
		SET_FLAG(hw, HW_MULTIALL_EN);
		CLR_FLAG(hw, HW_PROMISC_EN);
	} else {
		CLR_FLAG(hw, HW_MULTIALL_EN);
		CLR_FLAG(hw, HW_PROMISC_EN);
	}
	emac_hw_config_mac_ctrl(hw);

	/* update multicast address filtering */
	emac_hw_clear_mc_addr(hw);
	netdev_for_each_mc_addr(ha, netdev)
		emac_hw_set_mc_addr(hw, ha->addr);
}

/* Change MAC address */
static int emac_set_mac_address(struct net_device *netdev, void *p)
{
	struct emac_adapter *adpt = netdev_priv(netdev);
	struct emac_hw *hw = &adpt->hw;
	struct sockaddr *addr = p;

	if (!is_valid_ether_addr(addr->sa_data))
		return -EADDRNOTAVAIL;

	if (netif_running(netdev))
		return -EBUSY;

	memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
	memcpy(hw->mac_addr, addr->sa_data, netdev->addr_len);

	pm_runtime_get_sync(netdev->dev.parent);
	emac_hw_set_mac_addr(hw, hw->mac_addr);
	pm_runtime_mark_last_busy(netdev->dev.parent);
	pm_runtime_put_autosuspend(netdev->dev.parent);
	return 0;
}

/* Push the received skb to upper layers */
static void emac_receive_skb(struct emac_rx_queue *rxque,
			     struct sk_buff *skb,
			     u16 vlan_tag, bool vlan_flag)
{
	if (vlan_flag) {
		u16 vlan;

		EMAC_TAG_TO_VLAN(vlan_tag, vlan);
		__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan);
	}

	napi_gro_receive(&rxque->napi, skb);
}

/* Consume next received packet descriptor */
static bool emac_get_rrdesc(struct emac_rx_queue *rxque,
			    union emac_sw_rrdesc *srrd)
{
	struct emac_adapter *adpt = netdev_priv(rxque->netdev);
	u32 *hrrd = EMAC_RRD(rxque, adpt->rrdesc_size,
			     rxque->rrd.consume_idx);

	/* If time stamping is enabled, it will be added in the beginning of
	 * the hw rrd (hrrd). In sw rrd (srrd), dwords 4 & 5 are reserved for
	 * the time stamp; hence the conversion.
	 * Also, read the rrd word with update flag first; read rest of rrd
	 * only if update flag is set.
	 */
	if (adpt->tstamp_en)
		srrd->dfmt.dw[3] = *(hrrd + 5);
	else
		srrd->dfmt.dw[3] = *(hrrd + 3);
	rmb(); /* ensure hw receive returned descriptor timestamp is read */

	if (!srrd->genr.update)
		return false;

	if (adpt->tstamp_en) {
		srrd->dfmt.dw[4] = *(hrrd++);
		srrd->dfmt.dw[5] = *(hrrd++);
	} else {
		srrd->dfmt.dw[4] = 0;
		srrd->dfmt.dw[5] = 0;
	}

	srrd->dfmt.dw[0] = *(hrrd++);
	srrd->dfmt.dw[1] = *(hrrd++);
	srrd->dfmt.dw[2] = *(hrrd++);
	mb(); /* ensure descriptor is read */

	emac_dbg(adpt, rx_status, "RX[%d]:SRRD[%x]: %x:%x:%x:%x:%x:%x\n",
		 rxque->que_idx, rxque->rrd.consume_idx, srrd->dfmt.dw[0],
		 srrd->dfmt.dw[1], srrd->dfmt.dw[2], srrd->dfmt.dw[3],
		 srrd->dfmt.dw[4], srrd->dfmt.dw[5]);

	if (unlikely(srrd->genr.nor != 1)) {
		/* multiple rfd not supported */
		emac_err(adpt, "Multi rfd not support yet! nor = %d\n",
			 srrd->genr.nor);
	}

	/* mark rrd as processed */
	srrd->genr.update = 0;
	*hrrd = srrd->dfmt.dw[3];

	if (++rxque->rrd.consume_idx == rxque->rrd.count)
		rxque->rrd.consume_idx = 0;

	return true;
}

/* Produce new receive free descriptor */
static bool emac_set_rfdesc(struct emac_rx_queue *rxque,
			    union emac_sw_rfdesc *srfd)
{
	struct emac_adapter *adpt = netdev_priv(rxque->netdev);
	u32 *hrfd = EMAC_RFD(rxque, adpt->rfdesc_size,
			     rxque->rfd.produce_idx);

	*(hrfd++) = srfd->dfmt.dw[0];
	*hrfd = srfd->dfmt.dw[1];

	if (++rxque->rfd.produce_idx == rxque->rfd.count)
		rxque->rfd.produce_idx = 0;

	return true;
}

/* Produce new transmit descriptor */
static bool emac_set_tpdesc(struct emac_tx_queue *txque,
			    union emac_sw_tpdesc *stpd)
{
	struct emac_adapter *adpt = netdev_priv(txque->netdev);
	u32 *htpd;

	txque->tpd.last_produce_idx = txque->tpd.produce_idx;
	htpd = EMAC_TPD(txque, adpt->tpdesc_size, txque->tpd.produce_idx);

	if (++txque->tpd.produce_idx == txque->tpd.count)
		txque->tpd.produce_idx = 0;

	*(htpd++) = stpd->dfmt.dw[0];
	*(htpd++) = stpd->dfmt.dw[1];
	*(htpd++) = stpd->dfmt.dw[2];
	*htpd = stpd->dfmt.dw[3];

	emac_dbg(adpt, tx_done, "TX[%d]:STPD[%x]: %x:%x:%x:%x\n",
		 txque->que_idx, txque->tpd.last_produce_idx, stpd->dfmt.dw[0],
		 stpd->dfmt.dw[1], stpd->dfmt.dw[2], stpd->dfmt.dw[3]);

	return true;
}

/* Mark the last transmit descriptor as such (for the transmit packet) */
static void emac_set_tpdesc_lastfrag(struct emac_tx_queue *txque)
{
	struct emac_adapter *adpt = netdev_priv(txque->netdev);
	u32 tmp_tpd;
	u32 *htpd = EMAC_TPD(txque, adpt->tpdesc_size,
			     txque->tpd.last_produce_idx);

	tmp_tpd = *(htpd + 1);
	tmp_tpd |= EMAC_TPD_LAST_FRAGMENT;
	*(htpd + 1) = tmp_tpd;
}

void emac_set_tpdesc_tstamp_sav(struct emac_tx_queue *txque)
{
	struct emac_adapter *adpt = netdev_priv(txque->netdev);
	u32 tmp_tpd;
	u32 *htpd = EMAC_TPD(txque, adpt->tpdesc_size,
			     txque->tpd.last_produce_idx);

	tmp_tpd = *(htpd + 3);
	tmp_tpd |= EMAC_TPD_TSTAMP_SAVE;
	*(htpd + 3) = tmp_tpd;
}

/* Fill up receive queue's RFD with preallocated receive buffers */
static int emac_refresh_rx_buffer(struct emac_rx_queue *rxque)
{
	struct emac_adapter *adpt = netdev_priv(rxque->netdev);
	struct emac_hw *hw = &adpt->hw;
	struct emac_buffer *curr_rxbuf;
	struct emac_buffer *next_rxbuf;
	union emac_sw_rfdesc srfd;
	struct sk_buff *skb;
	void *skb_data = NULL;
	u32 count = 0;
	u32 next_produce_idx;

	next_produce_idx = rxque->rfd.produce_idx;
	if (++next_produce_idx == rxque->rfd.count)
		next_produce_idx = 0;
	curr_rxbuf = GET_RFD_BUFFER(rxque, rxque->rfd.produce_idx);
	next_rxbuf = GET_RFD_BUFFER(rxque, next_produce_idx);

	/* this always has a blank rx_buffer*/
	while (next_rxbuf->dma == 0) {
		skb = dev_alloc_skb(adpt->rxbuf_size + NET_IP_ALIGN);
		if (unlikely(!skb)) {
			emac_err(adpt, "alloc rx buffer failed\n");
			break;
		}

		/* Make buffer alignment 2 beyond a 16 byte boundary
		 * this will result in a 16 byte aligned IP header after
		 * the 14 byte MAC header is removed
		 */
		skb_reserve(skb, NET_IP_ALIGN);
		skb_data = skb->data;
		curr_rxbuf->skb = skb;
		curr_rxbuf->length = adpt->rxbuf_size;
		curr_rxbuf->dma = dma_map_single(rxque->dev, skb_data,
						 curr_rxbuf->length,
						 DMA_FROM_DEVICE);
		srfd.genr.addr = curr_rxbuf->dma;
		emac_set_rfdesc(rxque, &srfd);
		next_produce_idx = rxque->rfd.produce_idx;
		if (++next_produce_idx == rxque->rfd.count)
			next_produce_idx = 0;

		curr_rxbuf = GET_RFD_BUFFER(rxque, rxque->rfd.produce_idx);
		next_rxbuf = GET_RFD_BUFFER(rxque, next_produce_idx);
		count++;
	}

	if (count) {
		u32 prod_idx = (rxque->rfd.produce_idx << rxque->produce_shft) &
				rxque->produce_mask;
		wmb(); /* ensure that the descriptors are properly set */
		emac_reg_update32(hw, EMAC, rxque->produce_reg,
				  rxque->produce_mask, prod_idx);
		wmb(); /* ensure that the producer's index is flushed to HW */
		emac_dbg(adpt, rx_status, "RX[%d]: prod idx 0x%x\n",
			 rxque->que_idx, rxque->rfd.produce_idx);
	}

	return count;
}

static void emac_clean_rfdesc(struct emac_rx_queue *rxque,
			      union emac_sw_rrdesc *srrd)
{
	struct emac_buffer *rfbuf = rxque->rfd.rfbuff;
	u32 consume_idx = srrd->genr.si;
	u16 i;

	for (i = 0; i < srrd->genr.nor; i++) {
		rfbuf[consume_idx].skb = NULL;
		if (++consume_idx == rxque->rfd.count)
			consume_idx = 0;
	}

	rxque->rfd.consume_idx = consume_idx;
	rxque->rfd.process_idx = consume_idx;
}

static inline bool emac_skb_cb_expired(struct sk_buff *skb)
{
	if (time_is_after_jiffies(EMAC_SKB_CB(skb)->jiffies +
				  msecs_to_jiffies(100)))
		return false;
	return true;
}

/* proper lock must be acquired before polling */
static void emac_poll_hwtxtstamp(struct emac_adapter *adpt)
{
	struct sk_buff_head *pending_q = &adpt->hwtxtstamp_pending_queue;
	struct sk_buff_head *q = &adpt->hwtxtstamp_ready_queue;
	struct sk_buff *skb, *skb_tmp;
	struct emac_hwtxtstamp hwtxtstamp;

	while (emac_hw_read_tx_tstamp(&adpt->hw, &hwtxtstamp)) {
		bool found = false;

		adpt->hwtxtstamp_stats.rx++;

		skb_queue_walk_safe(pending_q, skb, skb_tmp) {
			if (EMAC_SKB_CB(skb)->tpd_idx == hwtxtstamp.ts_idx) {
				struct sk_buff *pskb;

				EMAC_HWTXTSTAMP_CB(skb)->sec = hwtxtstamp.sec;
				EMAC_HWTXTSTAMP_CB(skb)->ns = hwtxtstamp.ns;
				/* the tx timestamps for all the pending
				 * packets before this one are lost
				 */
				while ((pskb = __skb_dequeue(pending_q))
				       != skb) {
					EMAC_HWTXTSTAMP_CB(pskb)->sec = 0;
					EMAC_HWTXTSTAMP_CB(pskb)->ns = 0;
					__skb_queue_tail(q, pskb);
					adpt->hwtxtstamp_stats.lost++;
				}
				__skb_queue_tail(q, skb);
				found = true;
				break;
			}
		}

		if (!found) {
			emac_dbg(adpt, tx_done,
				 "no entry(tpd=%d) found, drop tx timestamp\n",
				 hwtxtstamp.ts_idx);
			adpt->hwtxtstamp_stats.drop++;
		}
	}

	skb_queue_walk_safe(pending_q, skb, skb_tmp) {
		/* No packet after this one expires */
		if (!emac_skb_cb_expired(skb))
			break;
		adpt->hwtxtstamp_stats.timeout++;
		emac_dbg(adpt, tx_done,
			 "tx timestamp timeout: tpd_idx=%d\n",
			 EMAC_SKB_CB(skb)->tpd_idx);

		__skb_unlink(skb, pending_q);
		EMAC_HWTXTSTAMP_CB(skb)->sec = 0;
		EMAC_HWTXTSTAMP_CB(skb)->ns = 0;
		__skb_queue_tail(q, skb);
	}
}

static void emac_schedule_hwtxtstamp_task(struct emac_adapter *adpt)
{
	if (TEST_FLAG(adpt, ADPT_STATE_DOWN))
		return;

	if (schedule_work(&adpt->hwtxtstamp_task))
		adpt->hwtxtstamp_stats.sched++;
}

static void emac_hwtxtstamp_task_routine(struct work_struct *work)
{
	struct emac_adapter *adpt = container_of(work, struct emac_adapter,
						 hwtxtstamp_task);
	struct sk_buff *skb;
	struct sk_buff_head q;
	unsigned long flags;

	adpt->hwtxtstamp_stats.poll++;

	__skb_queue_head_init(&q);

	while (1) {
		spin_lock_irqsave(&adpt->hwtxtstamp_lock, flags);
		if (adpt->hwtxtstamp_pending_queue.qlen)
			emac_poll_hwtxtstamp(adpt);
		skb_queue_splice_tail_init(&adpt->hwtxtstamp_ready_queue, &q);
		spin_unlock_irqrestore(&adpt->hwtxtstamp_lock, flags);

		if (!q.qlen)
			break;

		while ((skb = __skb_dequeue(&q))) {
			struct emac_hwtxtstamp_cb *cb = EMAC_HWTXTSTAMP_CB(skb);

			if (cb->sec || cb->ns) {
				struct skb_shared_hwtstamps ts;

				ts.hwtstamp = ktime_set(cb->sec, cb->ns);
				skb_tstamp_tx(skb, &ts);
				adpt->hwtxtstamp_stats.deliver++;
			}
			dev_kfree_skb_any(skb);
		}
	}

	if (adpt->hwtxtstamp_pending_queue.qlen)
		emac_schedule_hwtxtstamp_task(adpt);
}

/* Process receive event */
static void emac_handle_rx(struct emac_adapter *adpt,
			   struct emac_rx_queue *rxque,
			   int *num_pkts, int max_pkts)
{
	struct emac_hw *hw = &adpt->hw;
	struct net_device *netdev  = adpt->netdev;

	union emac_sw_rrdesc srrd;
	struct emac_buffer *rfbuf;
	struct sk_buff *skb;

	u32 hw_consume_idx, num_consume_pkts;
	u32 count = 0;
	u32 proc_idx;

	hw_consume_idx = emac_reg_field_r32(hw, EMAC, rxque->consume_reg,
					    rxque->consume_mask,
					    rxque->consume_shft);
	num_consume_pkts = (hw_consume_idx >= rxque->rrd.consume_idx) ?
		(hw_consume_idx -  rxque->rrd.consume_idx) :
		(hw_consume_idx + rxque->rrd.count - rxque->rrd.consume_idx);

	while (1) {
		if (!num_consume_pkts)
			break;

		if (!emac_get_rrdesc(rxque, &srrd))
			break;

		if (likely(srrd.genr.nor == 1)) {
			/* good receive */
			rfbuf = GET_RFD_BUFFER(rxque, srrd.genr.si);
			dma_unmap_single(rxque->dev, rfbuf->dma, rfbuf->length,
					 DMA_FROM_DEVICE);
			rfbuf->dma = 0;
			skb = rfbuf->skb;
		} else {
			/* multi rfd not supported */
			emac_err(adpt, "Multi rfd not support yet!\n");
			break;
		}
		emac_clean_rfdesc(rxque, &srrd);
		num_consume_pkts--;
		count++;

		/* Due to a HW issue in L4 check sum detection (UDP/TCP frags
		 * with DF set are marked as error), drop packets based on the
		 * error mask rather than the summary bit (ignoring L4F errors)
		 */
		if (srrd.dfmt.dw[EMAC_RRDES_STATS_DW_IDX] & EMAC_RRDES_ERROR) {
			emac_dbg(adpt, rx_status,
				 "Drop error packet[RRD: 0x%x:0x%x:0x%x:0x%x]\n",
				 srrd.dfmt.dw[0], srrd.dfmt.dw[1],
				 srrd.dfmt.dw[2], srrd.dfmt.dw[3]);

			dev_kfree_skb(skb);
			continue;
		}

		skb_put(skb, srrd.genr.pkt_len - ETH_FCS_LEN);
		skb->dev = netdev;
		skb->protocol = eth_type_trans(skb, skb->dev);
		if (netdev->features & NETIF_F_RXCSUM)
			skb->ip_summed = ((srrd.genr.l4f) ?
					  CHECKSUM_NONE : CHECKSUM_UNNECESSARY);
		else
			skb_checksum_none_assert(skb);

		if (TEST_FLAG(hw, HW_TS_RX_EN)) {
			struct skb_shared_hwtstamps *hwts = skb_hwtstamps(skb);

			hwts->hwtstamp = ktime_set(srrd.genr.ts_high,
						   srrd.genr.ts_low);
		}

		emac_receive_skb(rxque, skb, (u16)srrd.genr.cvlan_tag,
				 (bool)srrd.genr.cvlan_flag);

		netdev->last_rx = jiffies;
		(*num_pkts)++;
		if (*num_pkts >= max_pkts)
			break;
	}

	if (count) {
		proc_idx = (rxque->rfd.process_idx << rxque->process_shft) &
				rxque->process_mask;
		wmb(); /* ensure that the descriptors are properly cleared */
		emac_reg_update32(hw, EMAC, rxque->process_reg,
				  rxque->process_mask, proc_idx);
		wmb(); /* ensure that RFD producer index is flushed to HW */
		emac_dbg(adpt, rx_status, "RX[%d]: proc idx 0x%x\n",
			 rxque->que_idx, rxque->rfd.process_idx);

		emac_refresh_rx_buffer(rxque);
	}
}

/* get the number of free transmit descriptors */
static u32 emac_get_num_free_tpdescs(struct emac_tx_queue *txque)
{
	u32 produce_idx = txque->tpd.produce_idx;
	u32 consume_idx = txque->tpd.consume_idx;

	return (consume_idx > produce_idx) ?
		(consume_idx - produce_idx - 1) :
		(txque->tpd.count + consume_idx - produce_idx - 1);
}

/* Process transmit event */
static void emac_handle_tx(struct emac_adapter *adpt,
			   struct emac_tx_queue *txque)
{
	struct emac_hw *hw = &adpt->hw;
	struct emac_buffer *tpbuf;
	u32 hw_consume_idx;
	u32 pkts_compl = 0, bytes_compl = 0;

	hw_consume_idx = emac_reg_field_r32(hw, EMAC, txque->consume_reg,
					    txque->consume_mask,
					    txque->consume_shft);
	emac_dbg(adpt, tx_done, "TX[%d]: cons idx 0x%x\n",
		 txque->que_idx, hw_consume_idx);

	while (txque->tpd.consume_idx != hw_consume_idx) {
		tpbuf = GET_TPD_BUFFER(txque, txque->tpd.consume_idx);
		if (tpbuf->dma) {
			dma_unmap_single(txque->dev, tpbuf->dma, tpbuf->length,
					 DMA_TO_DEVICE);
			tpbuf->dma = 0;
		}

		if (tpbuf->skb) {
			pkts_compl++;
			bytes_compl += tpbuf->skb->len;
			dev_kfree_skb_irq(tpbuf->skb);
			tpbuf->skb = NULL;
		}

		if (++txque->tpd.consume_idx == txque->tpd.count)
			txque->tpd.consume_idx = 0;
	}

	if (pkts_compl || bytes_compl)
		netdev_completed_queue(adpt->netdev, pkts_compl, bytes_compl);
}

/* NAPI */
static int emac_napi_rtx(struct napi_struct *napi, int budget)
{
	struct emac_rx_queue *rxque = container_of(napi, struct emac_rx_queue,
						   napi);
	struct emac_adapter *adpt = netdev_priv(rxque->netdev);
	struct emac_irq_per_dev *irq = rxque->irq;
	struct emac_hw *hw = &adpt->hw;
	int work_done = 0;

	/* Keep link state information with original netdev */
	if (!netif_carrier_ok(adpt->netdev))
		goto quit_polling;

	emac_handle_rx(adpt, rxque, &work_done, budget);

	if (work_done < budget) {
quit_polling:
		napi_complete(napi);

		irq->mask |= rxque->intr;
		emac_reg_w32(hw, EMAC, emac_irq_cmn_tbl[irq->idx].mask_reg,
			     irq->mask);
		wmb(); /* ensure that interrupt enable is flushed to HW */
	}

	return work_done;
}

/* Check if enough transmit descriptors are available */
static bool emac_check_num_tpdescs(struct emac_tx_queue *txque,
				   const struct sk_buff *skb)
{
	u32 num_required = 1;
	u16 i;
	u16 proto_hdr_len = 0;

	if (skb_is_gso(skb)) {
		proto_hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
		if (proto_hdr_len < skb_headlen(skb))
			num_required++;
		if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6)
			num_required++;
	}

	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
		num_required++;

	return num_required < emac_get_num_free_tpdescs(txque);
}

/* Fill up transmit descriptors with TSO and Checksum offload information */
static int emac_tso_csum(struct emac_adapter *adpt,
			 struct emac_tx_queue *txque,
			 struct sk_buff *skb,
			 union emac_sw_tpdesc *stpd)
{
	u8  hdr_len;
	int retval;

	if (skb_is_gso(skb)) {
		if (skb_header_cloned(skb)) {
			retval = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
			if (unlikely(retval))
				return retval;
		}

		if (skb->protocol == htons(ETH_P_IP)) {
			u32 pkt_len =
				((unsigned char *)ip_hdr(skb) - skb->data) +
				ntohs(ip_hdr(skb)->tot_len);
			if (skb->len > pkt_len)
				pskb_trim(skb, pkt_len);
		}

		hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
		if (unlikely(skb->len == hdr_len)) {
			/* we only need to do csum */
			emac_warn(adpt, tx_err,
				  "tso not needed for packet with 0 data\n");
			goto do_csum;
		}

		if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) {
			ip_hdr(skb)->check = 0;
			tcp_hdr(skb)->check = ~csum_tcpudp_magic(
						ip_hdr(skb)->saddr,
						ip_hdr(skb)->daddr,
						0, IPPROTO_TCP, 0);
			stpd->genr.ipv4 = 1;
		}

		if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6) {
			/* ipv6 tso need an extra tpd */
			union emac_sw_tpdesc extra_tpd;

			memset(stpd, 0, sizeof(union emac_sw_tpdesc));
			memset(&extra_tpd, 0, sizeof(union emac_sw_tpdesc));

			ipv6_hdr(skb)->payload_len = 0;
			tcp_hdr(skb)->check = ~csum_ipv6_magic(
						&ipv6_hdr(skb)->saddr,
						&ipv6_hdr(skb)->daddr,
						0, IPPROTO_TCP, 0);
			extra_tpd.tso.pkt_len = skb->len;
			extra_tpd.tso.lso = 0x1;
			extra_tpd.tso.lso_v2 = 0x1;
			emac_set_tpdesc(txque, &extra_tpd);
			stpd->tso.lso_v2 = 0x1;
		}

		stpd->tso.lso = 0x1;
		stpd->tso.tcphdr_offset = skb_transport_offset(skb);
		stpd->tso.mss = skb_shinfo(skb)->gso_size;
		return 0;
	}

do_csum:
	if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) {
		u8 css, cso;

		cso = skb_transport_offset(skb);
		if (unlikely(cso & 0x1)) {
			emac_err(adpt, "payload offset should be even\n");
			return -EINVAL;
		}
		css = cso + skb->csum_offset;

		stpd->csum.payld_offset = cso >> 1;
		stpd->csum.cxsum_offset = css >> 1;
		stpd->csum.c_csum = 0x1;
	}

	return 0;
}

/* Fill up transmit descriptors */
static void emac_tx_map(struct emac_adapter *adpt,
			struct emac_tx_queue *txque,
			struct sk_buff *skb,
			union emac_sw_tpdesc *stpd)
{
	struct emac_hw  *hw = &adpt->hw;
	struct emac_buffer *tpbuf = NULL;
	u16 nr_frags = skb_shinfo(skb)->nr_frags;
	u32 len = skb_headlen(skb);
	u16 map_len = 0;
	u16 mapped_len = 0;
	u16 hdr_len = 0;
	u16 i;
	u32 tso = stpd->tso.lso;

	if (tso) {
		hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
		map_len = hdr_len;

		tpbuf = GET_TPD_BUFFER(txque, txque->tpd.produce_idx);
		tpbuf->length = map_len;
		tpbuf->dma = dma_map_single(txque->dev,	skb->data,
					    hdr_len, DMA_TO_DEVICE);
		mapped_len += map_len;
		stpd->genr.addr_lo = EMAC_DMA_ADDR_LO(tpbuf->dma);
		stpd->genr.addr_hi = EMAC_DMA_ADDR_HI(tpbuf->dma);
		stpd->genr.buffer_len = tpbuf->length;
		emac_set_tpdesc(txque, stpd);
	}

	if (mapped_len < len) {
		tpbuf = GET_TPD_BUFFER(txque, txque->tpd.produce_idx);
		tpbuf->length = len - mapped_len;
		tpbuf->dma = dma_map_single(txque->dev, skb->data + mapped_len,
					    tpbuf->length, DMA_TO_DEVICE);
		stpd->genr.addr_lo = EMAC_DMA_ADDR_LO(tpbuf->dma);
		stpd->genr.addr_hi = EMAC_DMA_ADDR_HI(tpbuf->dma);
		stpd->genr.buffer_len  = tpbuf->length;
		emac_set_tpdesc(txque, stpd);
	}

	for (i = 0; i < nr_frags; i++) {
		struct skb_frag_struct *frag;

		frag = &skb_shinfo(skb)->frags[i];

		tpbuf = GET_TPD_BUFFER(txque, txque->tpd.produce_idx);
		tpbuf->length = frag->size;
		tpbuf->dma = dma_map_page(txque->dev, frag->page.p,
					  frag->page_offset,
					  tpbuf->length,
					  DMA_TO_DEVICE);
		stpd->genr.addr_lo = EMAC_DMA_ADDR_LO(tpbuf->dma);
		stpd->genr.addr_hi = EMAC_DMA_ADDR_HI(tpbuf->dma);
		stpd->genr.buffer_len  = tpbuf->length;
		emac_set_tpdesc(txque, stpd);
	}

	/* The last tpd */
	emac_set_tpdesc_lastfrag(txque);

	if (TEST_FLAG(hw, HW_TS_TX_EN) &&
	    (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
		struct sk_buff *skb_ts = skb_clone(skb, GFP_ATOMIC);

		if (likely(skb_ts)) {
			unsigned long flags;

			emac_set_tpdesc_tstamp_sav(txque);
			skb_ts->sk = skb->sk;
			EMAC_SKB_CB(skb_ts)->tpd_idx =
				txque->tpd.last_produce_idx;
			EMAC_SKB_CB(skb_ts)->jiffies = get_jiffies_64();
			skb_shinfo(skb_ts)->tx_flags |= SKBTX_IN_PROGRESS;
			spin_lock_irqsave(&adpt->hwtxtstamp_lock, flags);
			if (adpt->hwtxtstamp_pending_queue.qlen >=
			    EMAC_TX_POLL_HWTXTSTAMP_THRESHOLD) {
				emac_poll_hwtxtstamp(adpt);
				adpt->hwtxtstamp_stats.tx_poll++;
			}
			__skb_queue_tail(&adpt->hwtxtstamp_pending_queue,
					 skb_ts);
			spin_unlock_irqrestore(&adpt->hwtxtstamp_lock, flags);
			adpt->hwtxtstamp_stats.tx++;
			emac_schedule_hwtxtstamp_task(adpt);
		}
	}

	/* The last buffer info contain the skb address,
	 * so it will be freed after unmap
	 */
	tpbuf->skb = skb;
}

/* Transmit the packet using specified transmit queue */
static int emac_start_xmit_frame(struct emac_adapter *adpt,
				 struct emac_tx_queue *txque,
				 struct sk_buff *skb)
{
	struct emac_hw  *hw = &adpt->hw;
	union emac_sw_tpdesc stpd;
	u32 prod_idx;

	if (TEST_FLAG(adpt, ADPT_STATE_DOWN)) {
		dev_kfree_skb_any(skb);
		return NETDEV_TX_OK;
	}

	if (!emac_check_num_tpdescs(txque, skb)) {
		/* not enough descriptors, just stop queue */
		netif_stop_queue(adpt->netdev);
		return NETDEV_TX_BUSY;
	}

	memset(&stpd, 0, sizeof(union emac_sw_tpdesc));

	if (emac_tso_csum(adpt, txque, skb, &stpd) != 0) {
		dev_kfree_skb_any(skb);
		return NETDEV_TX_OK;
	}

	if (vlan_tx_tag_present(skb)) {
		u16 vlan = vlan_tx_tag_get(skb);
		u16 tag;

		EMAC_VLAN_TO_TAG(vlan, tag);
		stpd.genr.cvlan_tag = tag;
		stpd.genr.ins_cvtag = 0x1;
	}

	if (skb_network_offset(skb) != ETH_HLEN)
		stpd.genr.type = 0x1;

	emac_tx_map(adpt, txque, skb, &stpd);

	netdev_sent_queue(adpt->netdev, skb->len);

	/* update produce idx */
	prod_idx = (txque->tpd.produce_idx << txque->produce_shft) &
			txque->produce_mask;
	wmb(); /* ensure that the descriptors are properly set */
	emac_reg_update32(hw, EMAC, txque->produce_reg,
			  txque->produce_mask, prod_idx);
	wmb(); /* ensure that RFD producer index is flushed to HW */
	emac_dbg(adpt, tx_queued, "TX[%d]: prod idx 0x%x\n",
		 txque->que_idx, txque->tpd.produce_idx);

	return NETDEV_TX_OK;
}

/* Transmit the packet */
static int emac_start_xmit(struct sk_buff *skb,
			   struct net_device *netdev)
{
	struct emac_adapter *adpt = netdev_priv(netdev);
	struct emac_tx_queue *txque;

	txque = &adpt->tx_queue[EMAC_ACTIVE_TXQ];
	return emac_start_xmit_frame(adpt, txque, skb);
}

/* This funciton aquire spin-lock so should not call from sleeping context */
static void emac_wol_gpio_irq(struct emac_adapter *adpt, bool enable)
{
	struct emac_irq_per_dev *wol_irq = &adpt->irq[EMAC_WOL_IRQ];
	unsigned long flags;

	spin_lock_irqsave(&adpt->wol_irq_lock, flags);
	if (enable && !adpt->is_wol_enabled)
		enable_irq(wol_irq->irq);
	else if (!enable && adpt->is_wol_enabled)
		disable_irq_nosync(wol_irq->irq);
	adpt->is_wol_enabled = enable;
	spin_unlock_irqrestore(&adpt->wol_irq_lock, flags);
}

/* ISR */
static irqreturn_t emac_wol_isr(int irq, void *data)
{
	struct emac_adapter *adpt = emac_irq_get_adpt(data);
	struct emac_phy *phy = &adpt->phy;
	struct net_device *netdev = adpt->netdev;
	int ret = 0;
	u16 val = 0;

	pm_runtime_get_sync(netdev->dev.parent);
	ret = emac_phy_read(adpt, phy->addr, MII_INT_STATUS, &val);
	pm_runtime_mark_last_busy(netdev->dev.parent);
	pm_runtime_put_autosuspend(netdev->dev.parent);

	if (!pm_runtime_status_suspended(adpt->netdev->dev.parent)) {
		if (!ret &&
		    ((val & LINK_SUCCESS_INTERRUPT) || (val & LINK_SUCCESS_BX)))
			emac_wol_gpio_irq(adpt, false);
	}
	return IRQ_HANDLED;
}

static irqreturn_t emac_isr(int _irq, void *data)
{
	struct emac_irq_per_dev *irq = data;
	const struct emac_irq_common *irq_cmn = &emac_irq_cmn_tbl[irq->idx];
	struct emac_adapter *adpt = emac_irq_get_adpt(data);
	struct emac_rx_queue *rxque = &adpt->rx_queue[irq->idx];
	struct emac_hw *hw = &adpt->hw;
	int max_ints = 1;
	u32 isr, status;

	emac_dbg(emac_irq_get_adpt(data), wol, "EMAC wol interrupt received\n");
	/* disable the interrupt */
	emac_reg_w32(hw, EMAC, irq_cmn->mask_reg, 0);
	wmb(); /* ensure that interrupt disable is flushed to HW */

	do {
		isr = emac_reg_r32(hw, EMAC, irq_cmn->status_reg);
		status = isr & irq->mask;

		if (status == 0)
			break;

		if (status & ISR_ERROR) {
			emac_warn(adpt, intr, "isr error status 0x%x\n",
				  status & ISR_ERROR);
			/* reset MAC */
			SET_FLAG(adpt, ADPT_TASK_REINIT_REQ);
			emac_task_schedule(adpt);
		}

		/* Schedule the napi for receive queue with interrupt
		 * status bit set
		 */
		if ((status & rxque->intr)) {
			if (napi_schedule_prep(&rxque->napi)) {
				irq->mask &= ~rxque->intr;
				__napi_schedule(&rxque->napi);
			}
		}

		if (status & ISR_TX_PKT) {
			if (status & TX_PKT_INT)
				emac_handle_tx(adpt, &adpt->tx_queue[0]);
			if (status & TX_PKT_INT1)
				emac_handle_tx(adpt, &adpt->tx_queue[1]);
			if (status & TX_PKT_INT2)
				emac_handle_tx(adpt, &adpt->tx_queue[2]);
			if (status & TX_PKT_INT3)
				emac_handle_tx(adpt, &adpt->tx_queue[3]);
		}

		if (status & ISR_OVER)
			emac_warn(adpt, intr, "TX/RX overflow status 0x%x\n",
				  status & ISR_OVER);

		/* link event */
		if (status & (ISR_GPHY_LINK | SW_MAN_INT)) {
			emac_check_lsc(adpt);
			break;
		}

		if (status & PTP_INT)
			emac_ptp_intr(hw);
	} while (--max_ints > 0);

	/* enable the interrupt */
	emac_reg_w32(hw, EMAC, irq_cmn->mask_reg, irq->mask);
	wmb(); /* ensure that interrupt enable is flushed to HW */
	return IRQ_HANDLED;
}

/* Enable interrupts */
static inline void emac_enable_intr(struct emac_adapter *adpt)
{
	struct emac_hw *hw = &adpt->hw;

	emac_hw_enable_intr(hw);
}

/* Disable interrupts */
static inline void emac_disable_intr(struct emac_adapter *adpt)
{
	struct emac_hw *hw = &adpt->hw;
	int i;

	emac_hw_disable_intr(hw);
	for (i = 0; i < EMAC_NUM_CORE_IRQ; i++)
		if (adpt->irq[i].irq)
			synchronize_irq(adpt->irq[i].irq);
}

/* Configure VLAN tag strip/insert feature */
static int emac_set_features(struct net_device *netdev,
			     netdev_features_t features)
{
	struct emac_adapter *adpt = netdev_priv(netdev);
	struct emac_hw *hw = &adpt->hw;
	netdev_features_t changed = features ^ netdev->features;

	if (!(changed & (NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX)))
		return 0;

	netdev->features = features;
	if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX)
		SET_FLAG(hw, HW_VLANSTRIP_EN);
	else
		CLR_FLAG(hw, HW_VLANSTRIP_EN);

	if (netif_running(netdev))
		emac_reinit_locked(adpt);

	return 0;
}

static void emac_napi_enable_all(struct emac_adapter *adpt)
{
	u8 i;

	for (i = 0; i < adpt->num_rxques; i++)
		napi_enable(&adpt->rx_queue[i].napi);
}

static void emac_napi_disable_all(struct emac_adapter *adpt)
{
	u8 i;

	for (i = 0; i < adpt->num_rxques; i++)
		napi_disable(&adpt->rx_queue[i].napi);
}

/* Free all descriptors of given transmit queue */
static void emac_clean_tx_queue(struct emac_tx_queue *txque)
{
	struct device *dev = txque->dev;
	unsigned long size;
	u32 i;

	/* ring already cleared, nothing to do */
	if (!txque->tpd.tpbuff)
		return;

	for (i = 0; i < txque->tpd.count; i++) {
		struct emac_buffer *tpbuf = GET_TPD_BUFFER(txque, i);

		if (tpbuf->dma) {
			dma_unmap_single(dev, tpbuf->dma, tpbuf->length,
					 DMA_TO_DEVICE);
			tpbuf->dma = 0;
		}
		if (tpbuf->skb) {
			dev_kfree_skb_any(tpbuf->skb);
			tpbuf->skb = NULL;
		}
	}

	size = sizeof(struct emac_buffer) * txque->tpd.count;
	memset(txque->tpd.tpbuff, 0, size);

	/* clear the descriptor ring */
	memset(txque->tpd.tpdesc, 0, txque->tpd.size);

	txque->tpd.consume_idx = 0;
	txque->tpd.produce_idx = 0;
}

static void emac_clean_all_tx_queues(struct emac_adapter *adpt)
{
	u8 i;

	for (i = 0; i < adpt->num_txques; i++)
		emac_clean_tx_queue(&adpt->tx_queue[i]);
	netdev_reset_queue(adpt->netdev);
}

/* Free all descriptors of given receive queue */
static void emac_clean_rx_queue(struct emac_rx_queue *rxque)
{
	struct device *dev = rxque->dev;
	unsigned long size;
	u32 i;

	/* ring already cleared, nothing to do */
	if (!rxque->rfd.rfbuff)
		return;

	for (i = 0; i < rxque->rfd.count; i++) {
		struct emac_buffer *rfbuf = GET_RFD_BUFFER(rxque, i);

		if (rfbuf->dma) {
			dma_unmap_single(dev, rfbuf->dma, rfbuf->length,
					 DMA_FROM_DEVICE);
			rfbuf->dma = 0;
		}
		if (rfbuf->skb) {
			dev_kfree_skb(rfbuf->skb);
			rfbuf->skb = NULL;
		}
	}

	size =  sizeof(struct emac_buffer) * rxque->rfd.count;
	memset(rxque->rfd.rfbuff, 0, size);

	/* clear the descriptor rings */
	memset(rxque->rrd.rrdesc, 0, rxque->rrd.size);
	rxque->rrd.produce_idx = 0;
	rxque->rrd.consume_idx = 0;

	memset(rxque->rfd.rfdesc, 0, rxque->rfd.size);
	rxque->rfd.produce_idx = 0;
	rxque->rfd.consume_idx = 0;
}

static void emac_clean_all_rx_queues(struct emac_adapter *adpt)
{
	u8 i;

	for (i = 0; i < adpt->num_rxques; i++)
		emac_clean_rx_queue(&adpt->rx_queue[i]);
}

/* Free all buffers associated with given transmit queue */
static void emac_free_tx_descriptor(struct emac_tx_queue *txque)
{
	emac_clean_tx_queue(txque);

	kfree(txque->tpd.tpbuff);
	txque->tpd.tpbuff = NULL;
	txque->tpd.tpdesc = NULL;
	txque->tpd.tpdma = 0;
	txque->tpd.size = 0;
}

static void emac_free_all_tx_descriptor(struct emac_adapter *adpt)
{
	u8 i;

	for (i = 0; i < adpt->num_txques; i++)
		emac_free_tx_descriptor(&adpt->tx_queue[i]);
}

/* Allocate TX descriptor ring for the given transmit queue */
static int emac_alloc_tx_descriptor(struct emac_adapter *adpt,
				    struct emac_tx_queue *txque)
{
	struct emac_ring_header *ring_header = &adpt->ring_header;
	unsigned long size;

	size = sizeof(struct emac_buffer) * txque->tpd.count;
	txque->tpd.tpbuff = kzalloc(size, GFP_KERNEL);
	if (!txque->tpd.tpbuff)
		goto err_alloc_tpq_buffer;

	txque->tpd.size = txque->tpd.count * (adpt->tpdesc_size * 4);
	txque->tpd.tpdma = ring_header->dma + ring_header->used;
	txque->tpd.tpdesc = ring_header->desc + ring_header->used;
	ring_header->used += ALIGN(txque->tpd.size, 8);
	txque->tpd.produce_idx = 0;
	txque->tpd.consume_idx = 0;
	return 0;

err_alloc_tpq_buffer:
	emac_err(adpt, "Unable to allocate memory for the Tx descriptor\n");
	return -ENOMEM;
}

static int emac_alloc_all_tx_descriptor(struct emac_adapter *adpt)
{
	int retval = 0;
	u8 i;

	for (i = 0; i < adpt->num_txques; i++) {
		retval = emac_alloc_tx_descriptor(adpt, &adpt->tx_queue[i]);
		if (retval)
			break;
	}

	if (retval) {
		emac_err(adpt, "Allocation for Tx Queue %u failed\n", i);
		for (i--; i > 0; i--)
			emac_free_tx_descriptor(&adpt->tx_queue[i]);
	}

	return retval;
}

/* Free all buffers associated with given transmit queue */
static void emac_free_rx_descriptor(struct emac_rx_queue *rxque)
{
	emac_clean_rx_queue(rxque);

	kfree(rxque->rfd.rfbuff);
	rxque->rfd.rfbuff = NULL;

	rxque->rfd.rfdesc = NULL;
	rxque->rfd.rfdma  = 0;
	rxque->rfd.size   = 0;

	rxque->rrd.rrdesc = NULL;
	rxque->rrd.rrdma  = 0;
	rxque->rrd.size   = 0;
}

static void emac_free_all_rx_descriptor(struct emac_adapter *adpt)
{
	u8 i;

	for (i = 0; i < adpt->num_rxques; i++)
		emac_free_rx_descriptor(&adpt->rx_queue[i]);
}

/* Allocate RX descriptor rings for the given receive queue */
static int emac_alloc_rx_descriptor(struct emac_adapter *adpt,
				    struct emac_rx_queue *rxque)
{
	struct emac_ring_header *ring_header = &adpt->ring_header;
	unsigned long size;

	size = sizeof(struct emac_buffer) * rxque->rfd.count;
	rxque->rfd.rfbuff = kzalloc(size, GFP_KERNEL);
	if (!rxque->rfd.rfbuff)
		goto err_alloc_rfq_buffer;

	rxque->rrd.size = rxque->rrd.count * (adpt->rrdesc_size * 4);
	rxque->rfd.size = rxque->rfd.count * (adpt->rfdesc_size * 4);

	rxque->rrd.rrdma = ring_header->dma + ring_header->used;
	rxque->rrd.rrdesc = ring_header->desc + ring_header->used;
	ring_header->used += ALIGN(rxque->rrd.size, 8);

	rxque->rfd.rfdma = ring_header->dma + ring_header->used;
	rxque->rfd.rfdesc = ring_header->desc + ring_header->used;
	ring_header->used += ALIGN(rxque->rfd.size, 8);

	rxque->rrd.produce_idx = 0;
	rxque->rrd.consume_idx = 0;

	rxque->rfd.produce_idx = 0;
	rxque->rfd.consume_idx = 0;

	return 0;

err_alloc_rfq_buffer:
	emac_err(adpt, "Unable to allocate memory for the Rx descriptor\n");
	return -ENOMEM;
}

static int emac_alloc_all_rx_descriptor(struct emac_adapter *adpt)
{
	int retval = 0;
	u8 i;

	for (i = 0; i < adpt->num_rxques; i++) {
		retval = emac_alloc_rx_descriptor(adpt, &adpt->rx_queue[i]);
		if (retval)
			break;
	}

	if (retval) {
		emac_err(adpt, "Allocation for Rx Queue %u failed\n", i);
		for (i--; i > 0; i--)
			emac_free_rx_descriptor(&adpt->rx_queue[i]);
	}

	return retval;
}

/* Allocate all TX and RX descriptor rings */
static int emac_alloc_all_rtx_descriptor(struct emac_adapter *adpt)
{
	struct emac_ring_header *ring_header = &adpt->ring_header;
	int num_tques = adpt->num_txques;
	int num_rques = adpt->num_rxques;
	unsigned int num_tx_descs = adpt->num_txdescs;
	unsigned int num_rx_descs = adpt->num_rxdescs;
	struct device *dev = adpt->rx_queue[0].dev;
	int retval, que_idx;

	for (que_idx = 0; que_idx < adpt->num_txques; que_idx++)
		adpt->tx_queue[que_idx].tpd.count = adpt->num_txdescs;

	for (que_idx = 0; que_idx < adpt->num_rxques; que_idx++) {
		adpt->rx_queue[que_idx].rrd.count = adpt->num_rxdescs;
		adpt->rx_queue[que_idx].rfd.count = adpt->num_rxdescs;
	}

	/* Ring DMA buffer. Each ring may need up to 8 bytes for alignment,
	 * hence the additional padding bytes are allocated.
	 */
	ring_header->size =
		num_tques * num_tx_descs * (adpt->tpdesc_size * 4) +
		num_rques * num_rx_descs * (adpt->rfdesc_size * 4) +
		num_rques * num_rx_descs * (adpt->rrdesc_size * 4) +
		num_tques * 8 + num_rques * 2 * 8;

	emac_info(adpt, ifup, "TX queues %d, TX descriptors %d\n",
		  num_tques, num_tx_descs);
	emac_info(adpt, ifup, "RX queues %d, Rx descriptors %d\n",
		  num_rques, num_rx_descs);

	ring_header->used = 0;
	ring_header->desc = dma_alloc_coherent(dev, ring_header->size,
					&ring_header->dma, GFP_KERNEL);
	if (!ring_header->desc) {
		emac_err(adpt, "dma_alloc_coherent failed\n");
		retval = -ENOMEM;
		goto err_alloc_dma;
	}
	memset(ring_header->desc, 0, ring_header->size);
	ring_header->used = ALIGN(ring_header->dma, 8) - ring_header->dma;

	retval = emac_alloc_all_tx_descriptor(adpt);
	if (retval)
		goto err_alloc_tx;

	retval = emac_alloc_all_rx_descriptor(adpt);
	if (retval)
		goto err_alloc_rx;

	return 0;

err_alloc_rx:
	emac_free_all_tx_descriptor(adpt);
err_alloc_tx:
	dma_free_coherent(dev, ring_header->size,
			  ring_header->desc, ring_header->dma);

	ring_header->desc = NULL;
	ring_header->dma  = 0;
	ring_header->size = 0;
	ring_header->used = 0;
err_alloc_dma:
	return retval;
}

/* Free all TX and RX descriptor rings */
static void emac_free_all_rtx_descriptor(struct emac_adapter *adpt)
{
	struct emac_ring_header *ring_header = &adpt->ring_header;
	struct device *dev = adpt->rx_queue[0].dev;

	emac_free_all_tx_descriptor(adpt);
	emac_free_all_rx_descriptor(adpt);

	dma_free_coherent(dev, ring_header->size,
			  ring_header->desc, ring_header->dma);

	ring_header->desc = NULL;
	ring_header->dma  = 0;
	ring_header->size = 0;
	ring_header->used = 0;
}

/* Initialize descriptor rings */
static void emac_init_ring_ptrs(struct emac_adapter *adpt)
{
	int i, j;

	for (i = 0; i < adpt->num_txques; i++) {
		struct emac_tx_queue *txque = &adpt->tx_queue[i];
		struct emac_buffer *tpbuf = txque->tpd.tpbuff;

		txque->tpd.produce_idx = 0;
		txque->tpd.consume_idx = 0;
		for (j = 0; j < txque->tpd.count; j++)
			tpbuf[j].dma = 0;
	}

	for (i = 0; i < adpt->num_rxques; i++) {
		struct emac_rx_queue *rxque = &adpt->rx_queue[i];
		struct emac_buffer *rfbuf = rxque->rfd.rfbuff;

		rxque->rrd.produce_idx = 0;
		rxque->rrd.consume_idx = 0;
		rxque->rfd.produce_idx = 0;
		rxque->rfd.consume_idx = 0;
		for (j = 0; j < rxque->rfd.count; j++)
			rfbuf[j].dma = 0;
	}
}

/* Configure Receive Side Scaling (RSS) */
static void emac_config_rss(struct emac_adapter *adpt)
{
	static const u8 key[40] = {
		0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
		0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
		0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
		0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
		0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA};

	struct emac_hw *hw = &adpt->hw;
	u32 reta = 0;
	u16 i, j;

	if (adpt->num_rxques == 1)
		return;

	if (!hw->rss_initialized) {
		hw->rss_initialized = true;
		/* initialize rss hash type and idt table size */
		hw->rss_hstype = EMAC_RSS_HSTYP_ALL_EN;
		hw->rss_idt_size = EMAC_RSS_IDT_SIZE;

		/* Fill out RSS key */
		memcpy(hw->rss_key, key, sizeof(hw->rss_key));

		/* Fill out redirection table */
		memset(hw->rss_idt, 0x0, sizeof(hw->rss_idt));
		for (i = 0, j = 0; i < EMAC_RSS_IDT_SIZE; i++, j++) {
			if (j == adpt->num_rxques)
				j = 0;
			if (j > 1)
				reta |= (j << ((i & 7) * 4));
			if ((i & 7) == 7) {
				hw->rss_idt[i>>3] = reta;
				reta = 0;
			}
		}
	}

	emac_hw_config_rss(hw);
}

/* Change the Maximum Transfer Unit (MTU) */
static int emac_change_mtu(struct net_device *netdev, int new_mtu)
{
	struct emac_adapter *adpt = netdev_priv(netdev);
	int old_mtu   = netdev->mtu;
	int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;

	if ((max_frame < EMAC_MIN_ETH_FRAME_SIZE) ||
	    (max_frame > EMAC_MAX_ETH_FRAME_SIZE)) {
		emac_err(adpt, "invalid MTU setting\n");
		return -EINVAL;
	}

	if ((old_mtu != new_mtu) && netif_running(netdev)) {
		emac_info(adpt, hw, "changing MTU from %d to %d\n",
			  netdev->mtu, new_mtu);
		netdev->mtu = new_mtu;
		adpt->hw.mtu = new_mtu;
		adpt->rxbuf_size = new_mtu > EMAC_DEF_RX_BUF_SIZE ?
			ALIGN(max_frame, 8) : EMAC_DEF_RX_BUF_SIZE;
		emac_reinit_locked(adpt);
	}

	return 0;
}

static inline int msm_emac_request_gpio_on(struct emac_adapter *adpt,
					   bool mdio, bool ephy)
{
	int i = 0;
	int result = 0;
	struct emac_phy *phy = &adpt->phy;

	if (phy->external) {
		for (i = 0; phy->uses_gpios && i < EMAC_GPIO_CNT; i++) {
			result = gpio_request(adpt->gpio[i], emac_gpio_name[i]);
			if (result) {
				emac_err(adpt,
					 "error:%d on gpio_request(%d:%s)\n",
					 result, adpt->gpio[i],
					 emac_gpio_name[i]);

				while (--i >= 0)
					gpio_free(adpt->gpio[i]);
				goto error;
			}
		}
	}
	return 0;
error:
	return result;
}

static inline int msm_emac_request_gpio_off(struct emac_adapter *adpt,
					    bool mdio, bool ephy)
{
	int i = 0;
	struct emac_phy *phy = &adpt->phy;

	if (phy->external) {
		for (i = 0; phy->uses_gpios && i < EMAC_GPIO_CNT; i++)
			gpio_free(adpt->gpio[i]);
	}
	return 0;
}

static inline int msm_emac_request_pinctrl_on(struct emac_adapter *adpt,
					      bool mdio, bool ephy)
{
	int result = 0;
	int ret    = 0;
	struct emac_phy *phy = &adpt->phy;

	if (phy->external) {
		if (mdio) {
			result = pinctrl_select_state(adpt->pinctrl,
						      adpt->mdio_pins_active);
			if (result)
				emac_err(adpt,
					 "error:%d Can not switch on %s pins\n",
					 result,
					 EMAC_PINCTRL_STATE_MDIO_ACTIVE);
			ret = result;
		}

		if (ephy) {
			result = pinctrl_select_state(adpt->pinctrl,
						      adpt->ephy_pins_active);
			if (result)
				emac_err(adpt,
					 "error:%d Can not switch on %s pins\n",
					 result,
					 EMAC_PINCTRL_STATE_EPHY_ACTIVE);
			if (!ret)
				ret = result;
		}
	}
	return ret;
}

static inline int msm_emac_request_pinctrl_off(struct emac_adapter *adpt,
					       bool mdio, bool ephy)
{
	int result = 0;
	int ret    = 0;
	struct emac_phy *phy = &adpt->phy;

	if (phy->external) {
		if (mdio) {
			result = pinctrl_select_state(adpt->pinctrl,
						      adpt->mdio_pins_sleep);
			if (result)
				emac_err(adpt,
					 "error:%d Can not switch off %s pins\n",
					 result, EMAC_PINCTRL_STATE_MDIO_SLEEP);
			ret = result;
		}

		if (ephy) {
			result = pinctrl_select_state(adpt->pinctrl,
						      adpt->ephy_pins_sleep);
			if (result)
				emac_err(adpt,
					 "error:%d Can not switch off %s pins\n",
					 result, EMAC_PINCTRL_STATE_EPHY_SLEEP);
			if (!ret)
				ret = result;
		}
	}
	return ret;
}

/* Bringup the interface/HW */
int emac_up(struct emac_adapter *adpt)
{
	struct emac_phy *phy = &adpt->phy;
	struct emac_hw *hw = &adpt->hw;
	struct net_device *netdev = adpt->netdev;
	int retval = 0;
	int i;

	if (!TEST_FLAG(adpt, ADPT_STATE_DOWN))
		return 0;

	emac_init_ring_ptrs(adpt);
	emac_set_rx_mode(netdev);

	emac_hw_config_mac(hw);
	emac_config_rss(adpt);

	retval = phy->ops.up(adpt);
	if (retval)
		return retval;

	for (i = 0; i < EMAC_NUM_CORE_IRQ; i++) {
		struct emac_irq_per_dev *irq = &adpt->irq[i];
		const struct emac_irq_common *irq_cmn = &emac_irq_cmn_tbl[i];

		if (!irq->irq)
			continue;

		retval = request_irq(irq->irq, irq_cmn->handler,
				     irq_cmn->irqflags, irq_cmn->name, irq);
		if (retval) {
			emac_err(adpt,
				 "error:%d on request_irq(%d:%s flags:0x%lx)\n",
				 retval, irq->irq, irq_cmn->name,
				 irq_cmn->irqflags);
			while (--i >= 0)
				if (adpt->irq[i].irq)
					free_irq(adpt->irq[i].irq,
						 &adpt->irq[i]);
			goto err_request_irq;
		}
	}

	for (i = 0; i < adpt->num_rxques; i++)
		emac_refresh_rx_buffer(&adpt->rx_queue[i]);

	emac_napi_enable_all(adpt);
	emac_enable_intr(adpt);

	netif_start_queue(netdev);
	CLR_FLAG(adpt, ADPT_STATE_DOWN);

	/* check link status */
	SET_FLAG(adpt, ADPT_TASK_LSC_REQ);
	adpt->link_jiffies = jiffies + EMAC_TRY_LINK_TIMEOUT;
	mod_timer(&adpt->emac_timer, jiffies);
	return retval;

err_request_irq:
	adpt->phy.ops.down(adpt);
	return retval;
}

/* Bring down the interface/HW */
void emac_down(struct emac_adapter *adpt, u32 ctrl)
{
	struct net_device *netdev = adpt->netdev;
	struct emac_phy *phy = &adpt->phy;
	struct emac_hw *hw = &adpt->hw;
	unsigned long flags;
	int i;

	if (TEST_FLAG(adpt, ADPT_STATE_DOWN))
		return;

	SET_FLAG(adpt, ADPT_STATE_DOWN);
	netif_stop_queue(netdev);

	netif_carrier_off(netdev);

	emac_disable_intr(adpt);
	emac_napi_disable_all(adpt);

	phy->ops.down(adpt);

	for (i = 0; i < EMAC_NUM_CORE_IRQ; i++)
		if (adpt->irq[i].irq)
			free_irq(adpt->irq[i].irq, &adpt->irq[i]);

	CLR_FLAG(adpt, ADPT_TASK_LSC_REQ);
	CLR_FLAG(adpt, ADPT_TASK_REINIT_REQ);
	CLR_FLAG(adpt, ADPT_TASK_CHK_SGMII_REQ);
	del_timer_sync(&adpt->emac_timer);

	cancel_work_sync(&adpt->hwtxtstamp_task);
	spin_lock_irqsave(&adpt->hwtxtstamp_lock, flags);
	__skb_queue_purge(&adpt->hwtxtstamp_pending_queue);
	__skb_queue_purge(&adpt->hwtxtstamp_ready_queue);
	spin_unlock_irqrestore(&adpt->hwtxtstamp_lock, flags);

	if (ctrl & EMAC_HW_CTRL_RESET_MAC)
		emac_hw_reset_mac(hw);

	phy->link_speed = EMAC_LINK_SPEED_UNKNOWN;
	emac_clean_all_tx_queues(adpt);
	emac_clean_all_rx_queues(adpt);
}

/* Called when the network interface is made active */
static int emac_open(struct net_device *netdev)
{
	struct emac_adapter *adpt = netdev_priv(netdev);
	struct emac_irq_per_dev *irq = &adpt->irq[EMAC_WOL_IRQ];
	const struct emac_irq_common *irq_cmn = &emac_irq_cmn_tbl[EMAC_WOL_IRQ];
	int retval;

	netif_carrier_off(netdev);

	/* allocate rx/tx dma buffer & descriptors */
	retval = emac_alloc_all_rtx_descriptor(adpt);
	if (retval) {
		emac_err(adpt, "error in emac_alloc_all_rtx_descriptor\n");
		goto err_alloc_rtx;
	}

	pm_runtime_get_sync(netdev->dev.parent);
	retval = emac_up(adpt);
	pm_runtime_mark_last_busy(netdev->dev.parent);
	pm_runtime_put_autosuspend(netdev->dev.parent);
	if (retval)
		goto err_up;

	/* Register for EMAC WOL ISR */
	retval = request_threaded_irq(irq->irq, NULL, irq_cmn->handler,
				      IRQF_TRIGGER_FALLING | IRQF_ONESHOT,
				      irq_cmn->name, irq);
	if (retval) {
		emac_err(adpt,
			 "error:%d on request_irq(%d:%s flags:0x%lx)\n",
			 retval, irq->irq, irq_cmn->name, irq_cmn->irqflags);
		goto err_up;
	} else {
		emac_wol_gpio_irq(adpt, false);
	}
	return retval;

err_up:
	emac_free_all_rtx_descriptor(adpt);
err_alloc_rtx:
	return retval;
}

/* Called when the network interface is disabled */
static int emac_close(struct net_device *netdev)
{
	struct emac_adapter *adpt = netdev_priv(netdev);
	struct emac_hw *hw = &adpt->hw;

	/* ensure no task is running and no reset is in progress */
	while (TEST_N_SET_FLAG(adpt, ADPT_STATE_RESETTING))
		msleep(20); /* Reset might take few 10s of ms */

	pm_runtime_get_sync(netdev->dev.parent);
	if (adpt->irq[EMAC_WOL_IRQ].irq)
		free_irq(adpt->irq[EMAC_WOL_IRQ].irq, &adpt->irq[EMAC_WOL_IRQ]);

	if (!TEST_FLAG(adpt, ADPT_STATE_DOWN))
		emac_down(adpt, EMAC_HW_CTRL_RESET_MAC);
	else
		emac_hw_reset_mac(hw);

	if (TEST_FLAG(hw, HW_PTP_CAP))
		emac_ptp_stop(hw);

	pm_runtime_mark_last_busy(netdev->dev.parent);
	pm_runtime_put_autosuspend(netdev->dev.parent);

	emac_free_all_rtx_descriptor(adpt);

	CLR_FLAG(adpt, ADPT_STATE_RESETTING);
	return 0;
}

/* Resize the descriptor rings */
int emac_resize_rings(struct net_device *netdev)
{
	/* close and then re-open interface */
	emac_close(netdev);
	return emac_open(netdev);
}

/* PHY related IOCTLs */
static int emac_mii_ioctl(struct net_device *netdev,
			  struct ifreq *ifr, int cmd)
{
	struct emac_adapter *adpt = netdev_priv(netdev);
	struct emac_phy *phy = &adpt->phy;
	struct mii_ioctl_data *data = if_mii(ifr);
	int retval = 0;

	switch (cmd) {
	case SIOCGMIIPHY:
		data->phy_id = phy->addr;
		break;

	case SIOCGMIIREG:
		if (!capable(CAP_NET_ADMIN)) {
			retval = -EPERM;
			break;
		}

		if (data->reg_num & ~(0x1F)) {
			retval = -EFAULT;
			break;
		}

		if (data->phy_id >= PHY_MAX_ADDR) {
			retval = -EFAULT;
			break;
		}

		if (phy->external && data->phy_id != phy->addr) {
			retval = -EFAULT;
			break;
		}

		retval = emac_phy_read(adpt, data->phy_id, data->reg_num,
				       &data->val_out);
		break;

	case SIOCSMIIREG:
		if (!capable(CAP_NET_ADMIN)) {
			retval = -EPERM;
			break;
		}

		if (data->reg_num & ~(0x1F)) {
			retval = -EFAULT;
			break;
		}

		if (data->phy_id >= PHY_MAX_ADDR) {
			retval = -EFAULT;
			break;
		}

		if (phy->external && data->phy_id != phy->addr) {
			retval = -EFAULT;
			break;
		}

		retval = emac_phy_write(adpt, data->phy_id, data->reg_num,
					data->val_in);

		break;
	}

	return retval;
}

/* IOCTL support for the interface */
static int emac_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
{
	struct emac_adapter *adpt = netdev_priv(netdev);
	struct emac_hw *hw = &adpt->hw;

	switch (cmd) {
	case SIOCGMIIPHY:
	case SIOCGMIIREG:
	case SIOCSMIIREG:
		return emac_mii_ioctl(netdev, ifr, cmd);
	case SIOCSHWTSTAMP:
		if (TEST_FLAG(hw, HW_PTP_CAP))
			return emac_tstamp_ioctl(netdev, ifr, cmd);
	default:
		return -EOPNOTSUPP;
	}
}

/* Read statistics information from the HW */
void emac_update_hw_stats(struct emac_adapter *adpt)
{
	u16 hw_reg_addr = 0;
	u64 *stats_item = NULL;
	u32 val;

	/* Prevent stats update while adapter is being reset, or if the
	 * connection is down.
	 */
	if (adpt->phy.link_speed == 0)
		return;

	if (TEST_FLAG(adpt, ADPT_STATE_DOWN) ||
	    TEST_FLAG(adpt, ADPT_STATE_RESETTING))
		return;

	/* update rx status */
	hw_reg_addr = REG_MAC_RX_STATUS_BIN;
	stats_item = &adpt->hw_stats.rx_ok;

	while (hw_reg_addr <= REG_MAC_RX_STATUS_END) {
		val = emac_reg_r32(&adpt->hw, EMAC, hw_reg_addr);
		*stats_item += val;
		stats_item++;
		hw_reg_addr += sizeof(u32);
	}

	/* additional rx status */
	val = emac_reg_r32(&adpt->hw, EMAC, EMAC_RXMAC_STATC_REG23);
	adpt->hw_stats.rx_crc_align += val;
	val = emac_reg_r32(&adpt->hw, EMAC, EMAC_RXMAC_STATC_REG24);
	adpt->hw_stats.rx_jubbers += val;

	/* update tx status */
	hw_reg_addr = REG_MAC_TX_STATUS_BIN;
	stats_item = &adpt->hw_stats.tx_ok;

	while (hw_reg_addr <= REG_MAC_TX_STATUS_END) {
		val = emac_reg_r32(&adpt->hw, EMAC, hw_reg_addr);
		*stats_item += val;
		stats_item++;
		hw_reg_addr += sizeof(u32);
	}

	/* additional tx status */
	val = emac_reg_r32(&adpt->hw, EMAC, EMAC_TXMAC_STATC_REG25);
	adpt->hw_stats.tx_col += val;
}

/* Provide network statistics info for the interface */
struct rtnl_link_stats64 *emac_get_stats64(struct net_device *netdev,
					   struct rtnl_link_stats64 *net_stats)
{
	struct emac_adapter *adpt = netdev_priv(netdev);
	struct emac_hw_stats *hw_stats = &adpt->hw_stats;

	memset(net_stats, 0, sizeof(struct rtnl_link_stats64));
	emac_update_hw_stats(adpt);
	net_stats->rx_packets = hw_stats->rx_ok;
	net_stats->tx_packets = hw_stats->tx_ok;
	net_stats->rx_bytes = hw_stats->rx_byte_cnt;
	net_stats->tx_bytes = hw_stats->tx_byte_cnt;
	net_stats->multicast = hw_stats->rx_mcast;
	net_stats->collisions = hw_stats->tx_1_col +
	    hw_stats->tx_2_col * 2 +
	    hw_stats->tx_late_col + hw_stats->tx_abort_col;

	net_stats->rx_errors = hw_stats->rx_frag + hw_stats->rx_fcs_err +
	    hw_stats->rx_len_err + hw_stats->rx_sz_ov +
	    hw_stats->rx_align_err;
	net_stats->rx_fifo_errors = hw_stats->rx_rxf_ov;
	net_stats->rx_length_errors = hw_stats->rx_len_err;
	net_stats->rx_crc_errors = hw_stats->rx_fcs_err;
	net_stats->rx_frame_errors = hw_stats->rx_align_err;
	net_stats->rx_over_errors = hw_stats->rx_rxf_ov;
	net_stats->rx_missed_errors = hw_stats->rx_rxf_ov;

	net_stats->tx_errors = hw_stats->tx_late_col + hw_stats->tx_abort_col +
	    hw_stats->tx_underrun + hw_stats->tx_trunc;
	net_stats->tx_fifo_errors = hw_stats->tx_underrun;
	net_stats->tx_aborted_errors = hw_stats->tx_abort_col;
	net_stats->tx_window_errors = hw_stats->tx_late_col;

	return net_stats;
}

static const struct net_device_ops emac_netdev_ops = {
	.ndo_open		= &emac_open,
	.ndo_stop		= &emac_close,
	.ndo_validate_addr	= &eth_validate_addr,
	.ndo_start_xmit		= &emac_start_xmit,
	.ndo_set_mac_address	= &emac_set_mac_address,
	.ndo_change_mtu		= &emac_change_mtu,
	.ndo_do_ioctl		= &emac_ioctl,
	.ndo_tx_timeout		= &emac_tx_timeout,
	.ndo_get_stats64	= &emac_get_stats64,
	.ndo_set_features       = emac_set_features,
	.ndo_set_rx_mode        = emac_set_rx_mode,
};

/* Reinitialize the interface/HW if required */
static void emac_reinit_task_routine(struct emac_adapter *adpt)
{
	if (!TEST_FLAG(adpt, ADPT_TASK_REINIT_REQ))
		return;
	CLR_FLAG(adpt, ADPT_TASK_REINIT_REQ);

	if (TEST_FLAG(adpt, ADPT_STATE_DOWN) ||
	    TEST_FLAG(adpt, ADPT_STATE_RESETTING))
		return;

	emac_reinit_locked(adpt);
}

static inline char *emac_get_link_speed_desc(u32 speed)
{
	switch (speed) {
	case EMAC_LINK_SPEED_1GB_FULL:
		return  "1 Gbps Duplex Full";
	case EMAC_LINK_SPEED_100_FULL:
		return "100 Mbps Duplex Full";
	case EMAC_LINK_SPEED_100_HALF:
		return "100 Mbps Duplex Half";
	case EMAC_LINK_SPEED_10_FULL:
		return "10 Mbps Duplex Full";
	case EMAC_LINK_SPEED_10_HALF:
		return "10 Mbps Duplex HALF";
	default:
		return "unknown speed";
	}
}

/* Check link status and handle link state changes */
static void emac_link_task_routine(struct emac_adapter *adpt)
{
	struct net_device *netdev = adpt->netdev;
	struct emac_phy *phy = &adpt->phy;
	struct emac_hw *hw = &adpt->hw;
	char *link_desc;

	if (!TEST_FLAG(adpt, ADPT_TASK_LSC_REQ))
		return;
	CLR_FLAG(adpt, ADPT_TASK_LSC_REQ);

	/* ensure that no reset is in progress while link task is running */
	while (TEST_N_SET_FLAG(adpt, ADPT_STATE_RESETTING))
		msleep(20); /* Reset might take few 10s of ms */

	if (TEST_FLAG(adpt, ADPT_STATE_DOWN))
		goto link_task_done;

	pm_runtime_get_sync(netdev->dev.parent);
	emac_phy_check_link(adpt, &phy->link_speed, &phy->link_up);
	link_desc = emac_get_link_speed_desc(phy->link_speed);
	pm_runtime_mark_last_busy(netdev->dev.parent);
	pm_runtime_put_autosuspend(netdev->dev.parent);

	if (phy->link_up) {
		if (netif_carrier_ok(netdev))
			goto link_task_done;

		if (!adpt->runtime_enable)
			pm_runtime_get_sync(netdev->dev.parent);
		/* Acquire wake lock if link is detected to avoid device going
		 * into suspend
		 */
		__pm_stay_awake(&adpt->link_wlock);
		emac_info(adpt, timer, "NIC Link is Up %s\n", link_desc);

		phy->ops.tx_clk_set_rate(adpt);

		emac_hw_start_mac(hw);
		netif_carrier_on(netdev);
		netif_wake_queue(netdev);
		adpt->runtime_enable = 1;
	} else {
		if (time_after(adpt->link_jiffies, jiffies))
			SET_FLAG(adpt, ADPT_TASK_LSC_REQ);

		/* only continue if link was up previously */
		if (!netif_carrier_ok(netdev))
			goto link_task_done;

		phy->link_speed = 0;
		emac_info(adpt, timer, "NIC Link is Down\n");
		netif_stop_queue(netdev);
		netif_carrier_off(netdev);

		emac_hw_stop_mac(hw);
		/* Release wake lock if link is disconnected */
		__pm_relax(&adpt->link_wlock);
		pm_runtime_mark_last_busy(netdev->dev.parent);
		pm_runtime_put_autosuspend(netdev->dev.parent);
		adpt->runtime_enable = 0;
	}

	/* link state transition, kick timer */
	mod_timer(&adpt->emac_timer, jiffies);

link_task_done:
	CLR_FLAG(adpt, ADPT_STATE_RESETTING);
}

/* Watchdog task routine */
static void emac_task_routine(struct work_struct *work)
{
	struct emac_adapter *adpt = container_of(work, struct emac_adapter,
						 emac_task);

	if (!TEST_FLAG(adpt, ADPT_STATE_WATCH_DOG))
		emac_warn(adpt, timer, "flag STATE_WATCH_DOG doesn't set\n");

	emac_reinit_task_routine(adpt);

	emac_link_task_routine(adpt);

	adpt->phy.ops.periodic_task(adpt);

	CLR_FLAG(adpt, ADPT_STATE_WATCH_DOG);
}

/* Timer routine */
static void emac_timer_routine(unsigned long data)
{
	struct emac_adapter *adpt = (struct emac_adapter *)data;
	unsigned long delay;

	if (pm_runtime_status_suspended(adpt->netdev->dev.parent))
		return;

	/* poll faster when waiting for link */
	if (TEST_FLAG(adpt, ADPT_TASK_LSC_REQ))
		delay = HZ / 10;
	else
		delay = 2 * HZ;

	/* Reset the timer */
	mod_timer(&adpt->emac_timer, delay + jiffies);

	emac_task_schedule(adpt);
}

/* Initialize all queue data structures */
static void emac_init_rtx_queues(struct platform_device *pdev,
				 struct emac_adapter *adpt)
{
	int que_idx;

	adpt->num_txques = EMAC_DEF_TX_QUEUES;
	adpt->num_rxques = EMAC_DEF_RX_QUEUES;

	for (que_idx = 0; que_idx < adpt->num_txques; que_idx++) {
		struct emac_tx_queue *txque = &adpt->tx_queue[que_idx];

		txque->que_idx = que_idx;
		txque->netdev  = adpt->netdev;
		txque->dev     = &pdev->dev;
	}

	for (que_idx = 0; que_idx < adpt->num_rxques; que_idx++) {
		struct emac_rx_queue *rxque = &adpt->rx_queue[que_idx];

		rxque->que_idx = que_idx;
		rxque->netdev  = adpt->netdev;
		rxque->dev     = &pdev->dev;
	}

	switch (adpt->num_rxques) {
	case 4:
		adpt->rx_queue[3].produce_reg = EMAC_MAILBOX_13;
		adpt->rx_queue[3].produce_mask = RFD3_PROD_IDX_BMSK;
		adpt->rx_queue[3].produce_shft = RFD3_PROD_IDX_SHFT;

		adpt->rx_queue[3].process_reg = EMAC_MAILBOX_13;
		adpt->rx_queue[3].process_mask = RFD3_PROC_IDX_BMSK;
		adpt->rx_queue[3].process_shft = RFD3_PROC_IDX_SHFT;

		adpt->rx_queue[3].consume_reg = EMAC_MAILBOX_8;
		adpt->rx_queue[3].consume_mask = RFD3_CONS_IDX_BMSK;
		adpt->rx_queue[3].consume_shft = RFD3_CONS_IDX_SHFT;

		adpt->rx_queue[3].irq = &adpt->irq[3];
		adpt->rx_queue[3].intr = adpt->irq[3].mask & ISR_RX_PKT;

		/* fall through */
	case 3:
		adpt->rx_queue[2].produce_reg = EMAC_MAILBOX_6;
		adpt->rx_queue[2].produce_mask = RFD2_PROD_IDX_BMSK;
		adpt->rx_queue[2].produce_shft = RFD2_PROD_IDX_SHFT;

		adpt->rx_queue[2].process_reg = EMAC_MAILBOX_6;
		adpt->rx_queue[2].process_mask = RFD2_PROC_IDX_BMSK;
		adpt->rx_queue[2].process_shft = RFD2_PROC_IDX_SHFT;

		adpt->rx_queue[2].consume_reg = EMAC_MAILBOX_7;
		adpt->rx_queue[2].consume_mask = RFD2_CONS_IDX_BMSK;
		adpt->rx_queue[2].consume_shft = RFD2_CONS_IDX_SHFT;

		adpt->rx_queue[2].irq = &adpt->irq[2];
		adpt->rx_queue[2].intr = adpt->irq[2].mask & ISR_RX_PKT;

		/* fall through */
	case 2:
		adpt->rx_queue[1].produce_reg = EMAC_MAILBOX_5;
		adpt->rx_queue[1].produce_mask = RFD1_PROD_IDX_BMSK;
		adpt->rx_queue[1].produce_shft = RFD1_PROD_IDX_SHFT;

		adpt->rx_queue[1].process_reg = EMAC_MAILBOX_5;
		adpt->rx_queue[1].process_mask = RFD1_PROC_IDX_BMSK;
		adpt->rx_queue[1].process_shft = RFD1_PROC_IDX_SHFT;

		adpt->rx_queue[1].consume_reg = EMAC_MAILBOX_7;
		adpt->rx_queue[1].consume_mask = RFD1_CONS_IDX_BMSK;
		adpt->rx_queue[1].consume_shft = RFD1_CONS_IDX_SHFT;

		adpt->rx_queue[1].irq = &adpt->irq[1];
		adpt->rx_queue[1].intr = adpt->irq[1].mask & ISR_RX_PKT;

		/* fall through */
	case 1:
		adpt->rx_queue[0].produce_reg = EMAC_MAILBOX_0;
		adpt->rx_queue[0].produce_mask = RFD0_PROD_IDX_BMSK;
		adpt->rx_queue[0].produce_shft = RFD0_PROD_IDX_SHFT;

		adpt->rx_queue[0].process_reg = EMAC_MAILBOX_0;
		adpt->rx_queue[0].process_mask = RFD0_PROC_IDX_BMSK;
		adpt->rx_queue[0].process_shft = RFD0_PROC_IDX_SHFT;

		adpt->rx_queue[0].consume_reg = EMAC_MAILBOX_3;
		adpt->rx_queue[0].consume_mask = RFD0_CONS_IDX_BMSK;
		adpt->rx_queue[0].consume_shft = RFD0_CONS_IDX_SHFT;

		adpt->rx_queue[0].irq = &adpt->irq[0];
		adpt->rx_queue[0].intr = adpt->irq[0].mask & ISR_RX_PKT;
		break;
	}

	switch (adpt->num_txques) {
	case 4:
		adpt->tx_queue[3].produce_reg = EMAC_MAILBOX_11;
		adpt->tx_queue[3].produce_mask = H3TPD_PROD_IDX_BMSK;
		adpt->tx_queue[3].produce_shft = H3TPD_PROD_IDX_SHFT;

		adpt->tx_queue[3].consume_reg = EMAC_MAILBOX_12;
		adpt->tx_queue[3].consume_mask = H3TPD_CONS_IDX_BMSK;
		adpt->tx_queue[3].consume_shft = H3TPD_CONS_IDX_SHFT;

		/* fall through */
	case 3:
		adpt->tx_queue[2].produce_reg = EMAC_MAILBOX_9;
		adpt->tx_queue[2].produce_mask = H2TPD_PROD_IDX_BMSK;
		adpt->tx_queue[2].produce_shft = H2TPD_PROD_IDX_SHFT;

		adpt->tx_queue[2].consume_reg = EMAC_MAILBOX_10;
		adpt->tx_queue[2].consume_mask = H2TPD_CONS_IDX_BMSK;
		adpt->tx_queue[2].consume_shft = H2TPD_CONS_IDX_SHFT;

		/* fall through */
	case 2:
		adpt->tx_queue[1].produce_reg = EMAC_MAILBOX_16;
		adpt->tx_queue[1].produce_mask = H1TPD_PROD_IDX_BMSK;
		adpt->tx_queue[1].produce_shft = H1TPD_PROD_IDX_SHFT;

		adpt->tx_queue[1].consume_reg = EMAC_MAILBOX_10;
		adpt->tx_queue[1].consume_mask = H1TPD_CONS_IDX_BMSK;
		adpt->tx_queue[1].consume_shft = H1TPD_CONS_IDX_SHFT;

		/* fall through */
	case 1:
		adpt->tx_queue[0].produce_reg = EMAC_MAILBOX_15;
		adpt->tx_queue[0].produce_mask = NTPD_PROD_IDX_BMSK;
		adpt->tx_queue[0].produce_shft = NTPD_PROD_IDX_SHFT;

		adpt->tx_queue[0].consume_reg = EMAC_MAILBOX_2;
		adpt->tx_queue[0].consume_mask = NTPD_CONS_IDX_BMSK;
		adpt->tx_queue[0].consume_shft = NTPD_CONS_IDX_SHFT;
		break;
	}
}

/* Initialize various data structures  */
static void emac_init_adapter(struct emac_adapter *adpt)
{
	struct emac_phy *phy = &adpt->phy;
	struct emac_hw *hw   = &adpt->hw;
	int max_frame;

	/* ids */
	hw->devid = (u16)emac_reg_field_r32(hw, EMAC, EMAC_DMA_MAS_CTRL,
					    DEV_ID_NUM_BMSK, DEV_ID_NUM_SHFT);
	hw->revid = (u16)emac_reg_field_r32(hw, EMAC, EMAC_DMA_MAS_CTRL,
					    DEV_REV_NUM_BMSK, DEV_REV_NUM_SHFT);

	/* descriptors */
	adpt->num_txdescs = EMAC_DEF_TX_DESCS;
	adpt->num_rxdescs = EMAC_DEF_RX_DESCS;

	/* mtu */
	adpt->netdev->mtu = ETH_DATA_LEN;
	hw->mtu = adpt->netdev->mtu;
	max_frame = adpt->netdev->mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
	adpt->rxbuf_size = adpt->netdev->mtu > EMAC_DEF_RX_BUF_SIZE ?
			   ALIGN(max_frame, 8) : EMAC_DEF_RX_BUF_SIZE;

	/* dma */
	hw->dma_order = emac_dma_ord_out;
	hw->dmar_block = emac_dma_req_4096;
	hw->dmaw_block = emac_dma_req_128;
	hw->dmar_dly_cnt = DMAR_DLY_CNT_DEF;
	hw->dmaw_dly_cnt = DMAW_DLY_CNT_DEF;
	hw->tpd_burst = TXQ0_NUM_TPD_PREF_DEF;
	hw->rfd_burst = RXQ0_NUM_RFD_PREF_DEF;

	/* link */
	phy->link_up = false;
	phy->link_speed = EMAC_LINK_SPEED_UNKNOWN;

	/* flow control */
	phy->req_fc_mode = EMAC_FC_FULL;
	phy->cur_fc_mode = EMAC_FC_FULL;
	phy->disable_fc_autoneg = false;

	/* rss */
	hw->rss_initialized = false;
	hw->rss_hstype = 0;
	hw->rss_idt_size = 0;
	hw->rss_base_cpu = 0;
	memset(hw->rss_idt, 0x0, sizeof(hw->rss_idt));
	memset(hw->rss_key, 0x0, sizeof(hw->rss_key));

	/* irq moderator */
	hw->irq_mod = ((EMAC_DEF_RX_IRQ_MOD / 2) << IRQ_MODERATOR2_INIT_SHFT) |
		      ((EMAC_DEF_TX_IRQ_MOD / 2) << IRQ_MODERATOR_INIT_SHFT);

	/* others */
	hw->preamble = EMAC_PREAMBLE_DEF;
	adpt->wol = EMAC_WOL_MAGIC | EMAC_WOL_PHY;
}

/* Get the clock */
static int emac_get_clk(struct platform_device *pdev,
			struct emac_adapter *adpt)
{
	struct clk *clk;
	u8 i;

	for (i = 0; i < EMAC_CLK_CNT; i++) {
		clk = clk_get(&pdev->dev, emac_clk_name[i]);

		if (IS_ERR(clk)) {
			emac_err(adpt, "error:%ld on clk_get(%s)\n",
				 PTR_ERR(clk), emac_clk_name[i]);

			while (--i >= 0)
				if (adpt->clk[i].clk)
					clk_put(adpt->clk[i].clk);
			return PTR_ERR(clk);
		}

		adpt->clk[i].clk = clk;
	}

	return 0;
}

/* Initialize clocks */
static int emac_init_clks(struct emac_adapter *adpt)
{
	int retval;

	retval = emac_clk_prepare_enable(adpt, EMAC_CLK_AXI);
	if (retval)
		return retval;

	retval = emac_clk_prepare_enable(adpt, EMAC_CLK_CFG_AHB);
	if (retval)
		return retval;

	retval = emac_clk_set_rate(adpt, EMAC_CLK_125M, EMC_CLK_RATE_19_2MHZ);
	if (retval)
		return retval;

	retval = emac_clk_prepare_enable(adpt, EMAC_CLK_125M);

	return retval;
}

/* Enable clocks; needs emac_init_clks to be called before */
static int emac_enable_clks(struct emac_adapter *adpt)
{
	int retval;

	retval = emac_clk_set_rate(adpt, EMAC_CLK_TX, EMC_CLK_RATE_125MHZ);
	if (retval)
		return retval;

	retval = emac_clk_prepare_enable(adpt, EMAC_CLK_TX);
	if (retval)
		return retval;

	retval = emac_clk_set_rate(adpt, EMAC_CLK_125M, EMC_CLK_RATE_125MHZ);
	if (retval)
		return retval;

	retval = emac_clk_set_rate(adpt, EMAC_CLK_SYS_25M, EMC_CLK_RATE_25MHZ);
	if (retval)
		return retval;

	retval = emac_clk_prepare_enable(adpt, EMAC_CLK_SYS_25M);
	if (retval)
		return retval;

	retval = emac_clk_prepare_enable(adpt, EMAC_CLK_RX);
	if (retval)
		return retval;

	retval = emac_clk_prepare_enable(adpt, EMAC_CLK_SYS);

	return retval;
}

/* Disable clocks */
static void emac_disable_clks(struct emac_adapter *adpt)
{
	u8 i;

	for (i = 0; i < EMAC_CLK_CNT; i++) {
		struct emac_clk *clk = &adpt->clk[i];

		if (clk->enabled) {
			clk_disable_unprepare(clk->clk);
			clk->enabled = false;
		}
	}
}

static int msm_emac_pinctrl_init(struct emac_adapter *adpt, struct device *dev)
{
	adpt->pinctrl = devm_pinctrl_get(dev);
	if (IS_ERR_OR_NULL(adpt->pinctrl)) {
		emac_dbg(adpt, probe, "error:%ld Failed to get pin ctrl\n",
			 PTR_ERR(adpt->pinctrl));
		return PTR_ERR(adpt->pinctrl);
	}
	adpt->mdio_pins_active = pinctrl_lookup_state(adpt->pinctrl,
				EMAC_PINCTRL_STATE_MDIO_ACTIVE);
	if (IS_ERR_OR_NULL(adpt->mdio_pins_active)) {
		emac_dbg(adpt, probe, "error:%ld Failed to lookup mdio pinctrl active state\n",
			 PTR_ERR(adpt->mdio_pins_active));
		return PTR_ERR(adpt->mdio_pins_active);
	}

	adpt->mdio_pins_sleep = pinctrl_lookup_state(adpt->pinctrl,
				EMAC_PINCTRL_STATE_MDIO_SLEEP);
	if (IS_ERR_OR_NULL(adpt->mdio_pins_sleep)) {
		emac_dbg(adpt, probe, "error:%ld Failed to lookup mdio pinctrl sleep state\n",
			 PTR_ERR(adpt->mdio_pins_sleep));
		return PTR_ERR(adpt->mdio_pins_sleep);
	}

	adpt->ephy_pins_active = pinctrl_lookup_state(adpt->pinctrl,
				EMAC_PINCTRL_STATE_EPHY_ACTIVE);
	if (IS_ERR_OR_NULL(adpt->ephy_pins_active)) {
		emac_dbg(adpt, probe, "error:%ld Failed to lookup ephy pinctrl active state\n",
			 PTR_ERR(adpt->ephy_pins_active));
		return PTR_ERR(adpt->ephy_pins_active);
	}

	adpt->ephy_pins_sleep = pinctrl_lookup_state(adpt->pinctrl,
				EMAC_PINCTRL_STATE_EPHY_SLEEP);
	if (IS_ERR_OR_NULL(adpt->ephy_pins_sleep)) {
		emac_dbg(adpt, probe, "error:%ld Failed to lookup ephy pinctrl sleep state\n",
			 PTR_ERR(adpt->ephy_pins_sleep));
		return PTR_ERR(adpt->ephy_pins_sleep);
	}

	return 0;
}

/* Get the resources */
static int emac_get_resources(struct platform_device *pdev,
			      struct emac_adapter *adpt)
{
	int retval = 0;
	u8 i;
	struct resource *res;
	struct net_device *netdev = adpt->netdev;
	struct device_node *node = pdev->dev.of_node;
	static const char * const res_name[] = {"emac", "emac_csr",
						"emac_1588"};
	const void *maddr;
	const struct of_device_id *id;

	if (!node)
		return -ENODEV;

	/* get id */
	retval = of_property_read_u32(node, "cell-index", &pdev->id);
	if (retval)
		return retval;

	/* get board id */
	id = of_match_node(emac_dt_match, node);
	if (id == NULL) {
		emac_err(adpt, "can't find emac_dt_match node\n");
		return -ENODEV;
	}
	adpt->phy.board_id = (enum emac_phy_map_type)id->data;

	/* get time stamp enable flag */
	adpt->tstamp_en = of_property_read_bool(node, "qcom,emac-tstamp-en");

	retval = msm_emac_pinctrl_init(adpt, &pdev->dev);
	if (!retval) {
		adpt->gpio_on = msm_emac_request_pinctrl_on;
		adpt->gpio_off = msm_emac_request_pinctrl_off;
	} else {
		for (i = 0; adpt->phy.uses_gpios && i < EMAC_GPIO_CNT; i++) {
			retval = of_get_named_gpio(node, emac_gpio_name[i], 0);
			if (retval < 0)
				return retval;

			adpt->gpio[i] = retval;
		}
		adpt->gpio_on = msm_emac_request_gpio_on;
		adpt->gpio_off = msm_emac_request_gpio_off;
	}

	/* get mac address */
	maddr = of_get_mac_address(node);
	if (maddr)
		memcpy(adpt->hw.mac_perm_addr, maddr, netdev->addr_len);

	/* get irqs */
	for (i = 0; i < EMAC_IRQ_CNT; i++) {
		retval = platform_get_irq_byname(pdev,
						 emac_irq_cmn_tbl[i].name);
		adpt->irq[i].irq = (retval > 0) ? retval : 0;
	}

	retval = emac_get_clk(pdev, adpt);
	if (retval)
		return retval;

	/* get register addresses */
	retval = 0;
	for (i = 0; i < NUM_EMAC_REG_BASES; i++) {
		/* 1588 is required only if tstamp is enabled */
		if ((i == EMAC_1588) && !adpt->tstamp_en)
			continue;

		res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
						   res_name[i]);
		if (!res) {
			emac_err(adpt, "can't get %s resource\n", res_name[i]);
			retval = -ENOMEM;
			break;
		}

		adpt->hw.reg_addr[i] = ioremap(res->start, resource_size(res));
		if (!adpt->hw.reg_addr[i]) {
			emac_err(adpt, "can't remap %s\n", res_name[i]);
			retval = -ENOMEM;
			break;
		}
	}

	if (retval) {
		while (--i >= 0)
			if (adpt->hw.reg_addr[i])
				iounmap(adpt->hw.reg_addr[i]);
		goto err_reg_res;
	}

	netdev->base_addr = (unsigned long)adpt->hw.reg_addr[EMAC];
	return 0;

err_reg_res:
	for (i = 0; i < EMAC_CLK_CNT; i++) {
		if (adpt->clk[i].clk)
			clk_put(adpt->clk[i].clk);
	}

	return retval;
}

/* Release resources */
static void emac_release_resources(struct emac_adapter *adpt)
{
	u8 i;

	if (ACPI_HANDLE(adpt->dev))
		return;

	for (i = 0; i < NUM_EMAC_REG_BASES; i++) {
		if (adpt->hw.reg_addr[i])
			iounmap(adpt->hw.reg_addr[i]);
	}

	for (i = 0; i < EMAC_CLK_CNT; i++) {
		if (adpt->clk[i].clk)
			clk_put(adpt->clk[i].clk);
	}
}

/* Get the regulator */
static int emac_get_regulator(struct platform_device *pdev,
			      struct emac_adapter *adpt)
{
	struct regulator *vreg;
	u8 i;
	int len = 0;
	u32 tmp[EMAC_VREG_CNT];

	for (i = 0; i < EMAC_VREG_CNT; i++) {
		vreg = devm_regulator_get(&pdev->dev, emac_regulator_name[i]);

		if (IS_ERR(vreg)) {
			emac_err(adpt, "error:%ld unable to get emac %s\n",
				 PTR_ERR(vreg), emac_regulator_name[i]);
			return PTR_ERR(vreg);
		}
		adpt->vreg[i].vreg = vreg;
	}

	if (of_get_property(pdev->dev.of_node,
			    "qcom,vdd-voltage-level", &len)) {
		if (len == sizeof(tmp)) {
			of_property_read_u32_array(pdev->dev.of_node,
						   "qcom,vdd-voltage-level",
						   tmp, len/sizeof(*tmp));

			for (i = 0; i < EMAC_VREG_CNT; i++)
				adpt->vreg[i].voltage_uv = tmp[i];
		} else {
			emac_err(adpt, "unable to read voltage values for emac LDOs\n");
			return -EINVAL;
		}
	} else {
		emac_err(adpt, "unable to read qcom,vdd-voltage-level emac dt property\n");
		return -EINVAL;
	}
	return 0;
}

/* Set the Voltage */
static int emac_set_voltage(struct emac_adapter *adpt, enum emac_vreg_id id,
			    int min_uV, int max_uV)
{
	int retval = regulator_set_voltage(adpt->vreg[id].vreg, min_uV, max_uV);
	if (retval)
		emac_err(adpt,
			 "error:%d set voltage for %s\n",
			 retval, emac_regulator_name[id]);
	return retval;
}

/* Enable the emac core, internal/external phy regulator */
static int emac_enable_regulator(struct emac_adapter *adpt, u8 start, u8 end)
{
	int retval = 0;
	u8 i;

	for (i = start; i <= end; i++) {
		if (adpt->vreg[i].voltage_uv) {
			retval = emac_set_voltage(adpt, i,
						  adpt->vreg[i].voltage_uv,
						  adpt->vreg[i].voltage_uv);
			if (retval)
				goto err;
		}

		retval = regulator_enable(adpt->vreg[i].vreg);
		if (retval) {
			emac_err(adpt, "error:%d enable regulator %s\n",
				 retval, emac_regulator_name[EMAC_VREG3]);
			goto err;
		} else {
			adpt->vreg[i].enabled = true;
		}
	}
err:
	return retval;
}

/* Disable the emac core, internal/external phy regulator */
static void emac_disable_regulator(struct emac_adapter *adpt, u8 start, u8 end)
{
	u8 i;

	for (i = start; i <= end; i++) {
		struct emac_regulator *vreg = &adpt->vreg[i];

		if (vreg->enabled) {
			regulator_disable(vreg->vreg);
			vreg->enabled = false;
		}

		if (adpt->vreg[i].voltage_uv) {
			emac_set_voltage(adpt, i,
					 0, adpt->vreg[i].voltage_uv);
		}
	}
}

/* LDO init */
static int msm_emac_ldo_init(struct platform_device *pdev,
			     struct emac_adapter *adpt)
{
	int retval = 0;

	retval = emac_get_regulator(pdev, adpt);
	if (retval)
		return retval;

	retval =  emac_enable_regulator(adpt, EMAC_VREG1, EMAC_VREG5);
	if (retval)
		return retval;
	return 0;
}

static int emac_pm_suspend(struct device *device, bool wol_enable)
{
	struct platform_device *pdev = to_platform_device(device);
	struct net_device *netdev = dev_get_drvdata(&pdev->dev);
	struct emac_adapter *adpt = netdev_priv(netdev);
	struct emac_hw *hw = &adpt->hw;
	struct emac_phy *phy = &adpt->phy;
	u32 wufc = adpt->wol;
	int retval = 0;

	/* Check link state. Don't suspend if link is up */
	if (netif_carrier_ok(adpt->netdev))
		return -EPERM;

	/* cannot suspend if WOL interrupt is not enabled */
	if (!adpt->irq[EMAC_WOL_IRQ].irq)
		return -EPERM;

	if (netif_running(netdev)) {
		/* ensure no task is running and no reset is in progress */
		while (TEST_N_SET_FLAG(adpt, ADPT_STATE_RESETTING))
			msleep(20);	/* Reset might take few 10s of ms */

		emac_down(adpt, 0);

		CLR_FLAG(adpt, ADPT_STATE_RESETTING);
	}

	emac_hw_config_pow_save(hw, adpt->phy.link_speed, !!wufc,
				!!(wufc & EMAC_WOL_MAGIC));

	/* Enable EPHY Link UP interrupt */
	if (!phy->link_up) {
		retval = emac_phy_write(adpt, phy->addr, MII_INT_ENABLE,
					LINK_SUCCESS_INTERRUPT |
					LINK_SUCCESS_BX);
		if (retval)
			return retval;

		if (wol_enable)
			emac_wol_gpio_irq(adpt, true);
	}

	adpt->gpio_off(adpt, true, false);
	emac_disable_clks(adpt);
	emac_disable_regulator(adpt, EMAC_VREG1, EMAC_VREG2);
	return 0;
}

static int emac_pm_resume(struct device *device)
{
	struct platform_device *pdev = to_platform_device(device);
	struct net_device *netdev = dev_get_drvdata(&pdev->dev);
	struct emac_adapter *adpt = netdev_priv(netdev);
	struct emac_phy *phy = &adpt->phy;
	struct emac_hw	*hw  = &adpt->hw;
	int retval = 0;

	adpt->gpio_on(adpt, true, false);
	emac_enable_regulator(adpt, EMAC_VREG1, EMAC_VREG2);
	emac_init_clks(adpt);
	emac_enable_clks(adpt);
	emac_hw_reset_mac(hw);
	emac_phy_reset_external(adpt);

	/* Disable EPHY Link UP interrupt */
	retval = emac_phy_write(adpt, phy->addr, MII_INT_ENABLE, 0);
	if (retval)
		goto error;


	if (netif_running(netdev)) {
		retval = emac_up(adpt);
		if (retval)
			goto error;
	}
	return 0;
error:
	return retval;
}

#ifdef CONFIG_PM_RUNTIME
static int emac_pm_runtime_suspend(struct device *device)
{
	return emac_pm_suspend(device, true);
}

static int emac_pm_runtime_resume(struct device *device)
{
	return emac_pm_resume(device);
}

static int emac_pm_runtime_idle(struct device *device)
{
	return 0;
}
#else
#define emac_pm_runtime_suspend NULL
#define emac_pm_runtime_resume	NULL
#define emac_pm_runtime_idle	NULL
#endif /* CONFIG_PM_RUNTIME */

#ifdef CONFIG_PM_SLEEP
static int emac_pm_sys_suspend(struct device *device)
{
	struct platform_device *pdev = to_platform_device(device);
	struct net_device *netdev = dev_get_drvdata(&pdev->dev);
	struct emac_adapter *adpt = netdev_priv(netdev);
	struct emac_phy *phy = &adpt->phy;
	bool link_up = false;

	/* Check link state. Don't suspend if link is up */
	if (netif_carrier_ok(adpt->netdev))
		return -EPERM;

	phy->link_speed = EMAC_LINK_SPEED_10_HALF;
	phy->link_up = link_up;

	/* Disable EPHY WOL interrupt*/
	emac_wol_gpio_irq(adpt, false);

	if (!pm_runtime_enabled(device) || !pm_runtime_suspended(device)) {
		emac_pm_suspend(device, false);
		/* Synchronize runtime-pm and system-pm states:
		 * at this point we are already suspended. However, the
		 * runtime-PM framework still thinks that we are active.
		 * The three calls below let the runtime-PM know that we are
		 * suspended already without re-invoking the suspend callback
		 */
		pm_runtime_disable(netdev->dev.parent);
		pm_runtime_set_suspended(netdev->dev.parent);
		pm_runtime_enable(netdev->dev.parent);
	}

	netif_device_detach(netdev);
	return 0;
}

static int emac_pm_sys_resume(struct device *device)
{
	struct platform_device *pdev = to_platform_device(device);
	struct net_device *netdev = dev_get_drvdata(&pdev->dev);
	struct emac_adapter *adpt = netdev_priv(netdev);

	netif_device_attach(netdev);

	if (!pm_runtime_enabled(device) || !pm_runtime_suspended(device)) {
		/* if runtime PM callback was not invoked (when both runtime-pm
		 * and systme-pm are in transition concurrently)
		*/
		emac_pm_resume(device);
		pm_runtime_mark_last_busy(netdev->dev.parent);
		pm_request_autosuspend(netdev->dev.parent);
	}
	/* Enable EPHY WOL interrupt*/
	emac_wol_gpio_irq(adpt, true);
	return 0;
}
#endif

/* Probe function */
static int emac_probe(struct platform_device *pdev)
{
	struct net_device *netdev;
	struct emac_adapter *adpt;
	struct emac_phy *phy;
	struct emac_hw *hw;
	int retval;
	u8 i;
	u32 hw_ver;

	netdev = alloc_etherdev(sizeof(struct emac_adapter));
	if (netdev == NULL) {
		dev_err(&pdev->dev, "etherdev alloc failed\n");
		retval = -ENOMEM;
		goto err_alloc_netdev;
	}

	dev_set_drvdata(&pdev->dev, netdev);
	SET_NETDEV_DEV(netdev, &pdev->dev);

	adpt = netdev_priv(netdev);
	adpt->netdev = netdev;
	phy = &adpt->phy;
	hw = &adpt->hw;
	adpt->msg_enable = netif_msg_init(msm_emac_msglvl, EMAC_MSG_DEFAULT);

	adpt->dma_mask = DMA_BIT_MASK(32);
	pdev->dev.dma_mask = &adpt->dma_mask;
	pdev->dev.dma_parms = &adpt->dma_parms;
	pdev->dev.coherent_dma_mask = DMA_BIT_MASK(32);

	dma_set_max_seg_size(&pdev->dev, 65536);
	dma_set_seg_boundary(&pdev->dev, 0xffffffff);

	for (i = 0; i < EMAC_IRQ_CNT; i++) {
		adpt->irq[i].idx  = i;
		adpt->irq[i].mask = emac_irq_cmn_tbl[i].init_mask;
	}
	adpt->irq[0].mask |= (msm_emac_intr_ext ? IMR_EXTENDED_MASK :
			      IMR_NORMAL_MASK);

	if (ACPI_HANDLE(adpt->dev))
		retval = emac_acpi_get_resources(pdev, adpt);
	else
		retval = emac_get_resources(pdev, adpt);
	if (retval)
		goto err_res;

	retval = msm_emac_ldo_init(pdev, adpt);
	if (retval)
		goto err_ldo_init;

	/* initialize clocks */
	retval = emac_init_clks(adpt);
	if (retval)
		goto err_clk_init;

	hw_ver = emac_reg_r32(hw, EMAC, EMAC_CORE_HW_VERSION);

	netdev->watchdog_timeo = EMAC_WATCHDOG_TIME;
	netdev->irq = adpt->irq[0].irq;

	if (adpt->tstamp_en)
		adpt->rrdesc_size = EMAC_TS_RRDESC_SIZE;
	else
		adpt->rrdesc_size = EMAC_RRDESC_SIZE;

	adpt->tpdesc_size = EMAC_TPDESC_SIZE;
	adpt->rfdesc_size = EMAC_RFDESC_SIZE;

	if (adpt->tstamp_en)
		SET_FLAG(hw, HW_PTP_CAP);

	/* init netdev */
	netdev->netdev_ops = &emac_netdev_ops;

	emac_set_ethtool_ops(netdev);

	/* init adapter */
	emac_init_adapter(adpt);

	/* init phy */
	retval = emac_phy_config(pdev, adpt);
	if (retval)
		goto err_init_phy;

	/* enable clocks */
	retval = emac_enable_clks(adpt);
	if (retval)
		goto err_clk_en;

	/* Configure MDIO lines */
	retval = adpt->gpio_on(adpt, true, true);
	if (retval)
		goto err_init_mdio_gpio;

	/* init external phy */
	retval = emac_phy_init_external(adpt);
	if (retval)
		goto err_init_ephy;

	/* reset mac */
	emac_hw_reset_mac(hw);

	/* setup link to put it in a known good starting state */
	retval = emac_phy_setup_link(adpt, phy->autoneg_advertised, true,
				     !phy->disable_fc_autoneg);
	if (retval)
		goto err_phy_link;

	/* set mac address */
	memcpy(hw->mac_addr, hw->mac_perm_addr, netdev->addr_len);
	memcpy(netdev->dev_addr, hw->mac_addr, netdev->addr_len);
	emac_hw_set_mac_addr(hw, hw->mac_addr);

	/* disable emac core and phy regulator */
	emac_disable_regulator(adpt, EMAC_VREG1, EMAC_VREG2);
	emac_disable_clks(adpt);

	/* set hw features */
	netdev->features = NETIF_F_SG | NETIF_F_HW_CSUM | NETIF_F_RXCSUM |
			NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_HW_VLAN_CTAG_RX |
			NETIF_F_HW_VLAN_CTAG_TX;
	netdev->hw_features = netdev->features;

	netdev->vlan_features |= NETIF_F_SG | NETIF_F_HW_CSUM |
				 NETIF_F_TSO | NETIF_F_TSO6;

	setup_timer(&adpt->emac_timer, &emac_timer_routine,
		    (unsigned long)adpt);
	INIT_WORK(&adpt->emac_task, emac_task_routine);

	/* Initialize queues */
	emac_init_rtx_queues(pdev, adpt);

	for (i = 0; i < adpt->num_rxques; i++)
		netif_napi_add(netdev, &adpt->rx_queue[i].napi,
			       emac_napi_rtx, 64);

	spin_lock_init(&adpt->hwtxtstamp_lock);
	spin_lock_init(&adpt->wol_irq_lock);
	skb_queue_head_init(&adpt->hwtxtstamp_pending_queue);
	skb_queue_head_init(&adpt->hwtxtstamp_ready_queue);
	INIT_WORK(&adpt->hwtxtstamp_task, emac_hwtxtstamp_task_routine);
	wakeup_source_init(&adpt->link_wlock, dev_name(&pdev->dev));

	SET_FLAG(hw, HW_VLANSTRIP_EN);
	SET_FLAG(adpt, ADPT_STATE_DOWN);
	strlcpy(netdev->name, "eth%d", sizeof(netdev->name));

	adpt->runtime_enable = 0;
	pm_runtime_set_autosuspend_delay(&pdev->dev, EMAC_TRY_LINK_TIMEOUT);
	pm_runtime_use_autosuspend(&pdev->dev);
	pm_runtime_enable(&pdev->dev);

	/* if  !CONFIG_PM_RUNTIME then enable all the resources here and mange
	 * resources from system suspend/resume callbacks
	 */
	if (!pm_runtime_enabled(&pdev->dev))
		emac_pm_resume(&pdev->dev);

	retval = register_netdev(netdev);
	if (retval) {
		emac_err(adpt, "register netdevice failed\n");
		goto err_register_netdev;
	}

	/* carrier off reporting is important to ethtool even BEFORE open */
	netif_carrier_off(netdev);

	if (TEST_FLAG(hw, HW_PTP_CAP)) {
		pm_runtime_get_sync(&pdev->dev);
		emac_ptp_init(adpt->netdev);
		pm_runtime_mark_last_busy(&pdev->dev);
		pm_runtime_put_autosuspend(&pdev->dev);
	}

	pr_info("%s - version %s\n", emac_drv_description, emac_drv_version);
	emac_dbg(adpt, probe, "EMAC HW ID %d.%d\n", hw->devid, hw->revid);
	emac_dbg(adpt, probe, "EMAC HW version %d.%d.%d\n",
		 (hw_ver & MAJOR_BMSK) >> MAJOR_SHFT,
		 (hw_ver & MINOR_BMSK) >> MINOR_SHFT,
		 (hw_ver & STEP_BMSK) >> STEP_SHFT);
	return 0;

err_register_netdev:
err_phy_link:
err_init_ephy:
	adpt->gpio_off(adpt, true, true);
err_init_mdio_gpio:
err_clk_en:
err_init_phy:
err_clk_init:
	emac_disable_clks(adpt);
err_ldo_init:
	emac_disable_regulator(adpt, EMAC_VREG1, EMAC_VREG5);
	emac_release_resources(adpt);
err_res:
	free_netdev(netdev);
err_alloc_netdev:
	return retval;
}

static int emac_remove(struct platform_device *pdev)
{
	struct net_device *netdev = dev_get_drvdata(&pdev->dev);
	struct emac_adapter *adpt = netdev_priv(netdev);
	struct emac_hw *hw = &adpt->hw;

	pr_info("exiting %s\n", emac_drv_name);

	/* Disable EPHY WOL interrupt in suspend */
	emac_wol_gpio_irq(adpt, false);
	unregister_netdev(netdev);
	wakeup_source_trash(&adpt->link_wlock);
	if (TEST_FLAG(hw, HW_PTP_CAP))
		emac_ptp_remove(netdev);

	adpt->gpio_off(adpt, true, true);
	emac_disable_clks(adpt);
	emac_disable_regulator(adpt, EMAC_VREG1, EMAC_VREG5);

	emac_release_resources(adpt);
	free_netdev(netdev);
	dev_set_drvdata(&pdev->dev, NULL);
	return 0;
}

static const struct dev_pm_ops emac_pm_ops = {
	SET_SYSTEM_SLEEP_PM_OPS(
		emac_pm_sys_suspend,
		emac_pm_sys_resume
	)
	SET_RUNTIME_PM_OPS(
		emac_pm_runtime_suspend,
		emac_pm_runtime_resume,
		emac_pm_runtime_idle
	)
};

static struct of_device_id emac_dt_match[] = {
	{
		.compatible = "qcom,emac",
		.data = (void *)EMAC_PHY_MAP_DEFAULT,
	},
	{
		.compatible = "qcom,mdm9607-emac",
		.data = (void *)EMAC_PHY_MAP_MDM9607,
	},
	{}
};

static struct platform_driver emac_platform_driver = {
	.probe   = emac_probe,
	.remove  = emac_remove,
	.driver = {
		.owner = THIS_MODULE,
		.name	= "msm_emac",
		.pm = &emac_pm_ops,
		.of_match_table = emac_dt_match,
		.acpi_match_table = ACPI_PTR(emac_acpi_match),
	},
};

static int __init emac_init_module(void)
{
	return platform_driver_register(&emac_platform_driver);
}

static void __exit emac_exit_module(void)
{
	platform_driver_unregister(&emac_platform_driver);
}

module_init(emac_init_module);
module_exit(emac_exit_module);

MODULE_LICENSE("GPL");