/* Copyright (c) 2011-2013, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and * only version 2 as published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. */ #include /* u32 */ #include /* pr_info() */ #include /* mutex */ #include /* list_head */ #include /* kzalloc() */ #include /* request_irq() */ #include /* memset */ #include #include "sps_bam.h" #include "bam.h" #include "spsi.h" /* All BAM global IRQ sources */ #define BAM_IRQ_ALL (BAM_DEV_IRQ_HRESP_ERROR | BAM_DEV_IRQ_ERROR | \ BAM_DEV_IRQ_TIMER) /* BAM device state flags */ #define BAM_STATE_INIT (1UL << 1) #define BAM_STATE_IRQ (1UL << 2) #define BAM_STATE_ENABLED (1UL << 3) #define BAM_STATE_BAM2BAM (1UL << 4) #define BAM_STATE_MTI (1UL << 5) #define BAM_STATE_REMOTE (1UL << 6) /* Mask for valid hardware descriptor flags */ #define BAM_IOVEC_FLAG_MASK \ (SPS_IOVEC_FLAG_INT | SPS_IOVEC_FLAG_EOT | SPS_IOVEC_FLAG_EOB | \ SPS_IOVEC_FLAG_NWD | SPS_IOVEC_FLAG_CMD | SPS_IOVEC_FLAG_LOCK | \ SPS_IOVEC_FLAG_UNLOCK | SPS_IOVEC_FLAG_IMME) /* Mask for invalid BAM-to-BAM pipe options */ #define BAM2BAM_O_INVALID \ (SPS_O_DESC_DONE | \ SPS_O_EOT | \ SPS_O_POLL | \ SPS_O_NO_Q | \ SPS_O_ACK_TRANSFERS) /** * Pipe/client pointer value indicating pipe is allocated, but no client has * been assigned */ #define BAM_PIPE_UNASSIGNED ((struct sps_pipe *)0x77777777) /* Check whether pipe has been assigned */ #define BAM_PIPE_IS_ASSIGNED(p) \ (((p) != NULL) && ((p) != BAM_PIPE_UNASSIGNED)) /* Is MTI use supported for a specific BAM version? */ #define BAM_VERSION_MTI_SUPPORT(ver) (ver <= 2) /* Event option<->event translation table entry */ struct sps_bam_opt_event_table { enum sps_event event_id; enum sps_option option; enum bam_pipe_irq pipe_irq; }; static const struct sps_bam_opt_event_table opt_event_table[] = { {SPS_EVENT_EOT, SPS_O_EOT, BAM_PIPE_IRQ_EOT}, {SPS_EVENT_DESC_DONE, SPS_O_DESC_DONE, BAM_PIPE_IRQ_DESC_INT}, {SPS_EVENT_WAKEUP, SPS_O_WAKEUP, BAM_PIPE_IRQ_WAKE}, {SPS_EVENT_INACTIVE, SPS_O_INACTIVE, BAM_PIPE_IRQ_TIMER}, {SPS_EVENT_OUT_OF_DESC, SPS_O_OUT_OF_DESC, BAM_PIPE_IRQ_OUT_OF_DESC}, {SPS_EVENT_ERROR, SPS_O_ERROR, BAM_PIPE_IRQ_ERROR} }; /* Pipe event source handler */ static void pipe_handler(struct sps_bam *dev, struct sps_pipe *pipe); /** * Pipe transfer event (EOT, DESC_DONE) source handler. * This function is called by pipe_handler() and other functions to process the * descriptor FIFO. */ static void pipe_handler_eot(struct sps_bam *dev, struct sps_pipe *pipe); /** * BAM driver initialization */ int sps_bam_driver_init(u32 options) { int n; /* * Check that SPS_O_ and BAM_PIPE_IRQ_ values are identical. * This is required so that the raw pipe IRQ status can be passed * to the client in the SPS_EVENT_IRQ. */ for (n = 0; n < ARRAY_SIZE(opt_event_table); n++) { if ((u32)opt_event_table[n].option != (u32)opt_event_table[n].pipe_irq) { SPS_ERR("sps:SPS_O 0x%x != HAL IRQ 0x%x\n", opt_event_table[n].option, opt_event_table[n].pipe_irq); return SPS_ERROR; } } return 0; } /** * BAM interrupt service routine * * This function is the BAM interrupt service routine. * * @ctxt - pointer to ISR's registered argument * * @return void */ static irqreturn_t bam_isr(int irq, void *ctxt) { struct sps_bam *dev = ctxt; struct sps_pipe *pipe; u32 source; unsigned long flags = 0; spin_lock_irqsave(&dev->isr_lock, flags); /* Get BAM interrupt source(s) */ if ((dev->state & BAM_STATE_MTI) == 0) { u32 mask = dev->pipe_active_mask; enum sps_callback_case cb_case; source = bam_check_irq_source(dev->base, dev->props.ee, mask, &cb_case); SPS_DBG1("sps:bam_isr:bam=0x%x;source=0x%x;mask=0x%x.\n", BAM_ID(dev), source, mask); if ((source & (1UL << 31)) && (dev->props.callback)) { SPS_DBG1("sps:bam_isr:bam=0x%x;callback for case %d.\n", BAM_ID(dev), cb_case); dev->props.callback(cb_case, dev->props.user); } /* Mask any non-local source */ source &= dev->pipe_active_mask; } else { /* If MTIs are used, must poll each active pipe */ source = dev->pipe_active_mask; SPS_DBG1("sps:bam_isr for MTI:bam=0x%x;source=0x%x.\n", BAM_ID(dev), source); } /* Process active pipe sources */ pipe = list_first_entry(&dev->pipes_q, struct sps_pipe, list); list_for_each_entry(pipe, &dev->pipes_q, list) { /* Check this pipe's bit in the source mask */ if (BAM_PIPE_IS_ASSIGNED(pipe) && (source & pipe->pipe_index_mask)) { /* This pipe has an interrupt pending */ pipe_handler(dev, pipe); source &= ~pipe->pipe_index_mask; } if (source == 0) break; } /* Process any inactive pipe sources */ if (source) { SPS_ERR("sps:IRQ from BAM 0x%x inactive pipe(s) 0x%x\n", BAM_ID(dev), source); dev->irq_from_disabled_pipe++; } spin_unlock_irqrestore(&dev->isr_lock, flags); return IRQ_HANDLED; } /** * BAM device enable */ int sps_bam_enable(struct sps_bam *dev) { u32 num_pipes; u32 irq_mask; int result; int rc; int MTIenabled; /* Is this BAM enabled? */ if ((dev->state & BAM_STATE_ENABLED)) return 0; /* Yes, so no work to do */ /* Is there any access to this BAM? */ if ((dev->props.manage & SPS_BAM_MGR_ACCESS_MASK) == SPS_BAM_MGR_NONE) { SPS_ERR("sps:No local access to BAM 0x%x\n", BAM_ID(dev)); return SPS_ERROR; } /* Set interrupt handling */ if ((dev->props.options & SPS_BAM_OPT_IRQ_DISABLED) != 0 || dev->props.irq == SPS_IRQ_INVALID) { /* Disable the BAM interrupt */ irq_mask = 0; dev->state &= ~BAM_STATE_IRQ; } else { /* Register BAM ISR */ if (dev->props.irq > 0) result = request_irq(dev->props.irq, (irq_handler_t) bam_isr, IRQF_TRIGGER_HIGH, "sps", dev); if (result) { SPS_ERR("sps:Failed to enable BAM 0x%x IRQ %d\n", BAM_ID(dev), dev->props.irq); return SPS_ERROR; } /* Enable the BAM interrupt */ irq_mask = BAM_IRQ_ALL; dev->state |= BAM_STATE_IRQ; /* Register BAM IRQ for apps wakeup */ if (dev->props.options & SPS_BAM_OPT_IRQ_WAKEUP) { result = enable_irq_wake(dev->props.irq); if (result) { SPS_ERR( "sps:Fail to enable wakeup irq for BAM 0x%x IRQ %d\n", BAM_ID(dev), dev->props.irq); return SPS_ERROR; } else SPS_DBG2( "sps:Enable wakeup irq for BAM 0x%x IRQ %d\n", BAM_ID(dev), dev->props.irq); } } /* Is global BAM control managed by the local processor? */ num_pipes = 0; if ((dev->props.manage & SPS_BAM_MGR_DEVICE_REMOTE) == 0) /* Yes, so initialize the BAM device */ rc = bam_init(dev->base, dev->props.ee, (u16) dev->props.summing_threshold, irq_mask, &dev->version, &num_pipes, dev->props.options); else /* No, so just verify that it is enabled */ rc = bam_check(dev->base, &dev->version, &num_pipes); if (rc) { SPS_ERR("sps:Fail to init BAM 0x%x IRQ %d\n", BAM_ID(dev), dev->props.irq); return SPS_ERROR; } /* Check if this BAM supports MTIs (Message Triggered Interrupts) or * multiple EEs (Execution Environments). * MTI and EE support are mutually exclusive. */ MTIenabled = BAM_VERSION_MTI_SUPPORT(dev->version); if ((dev->props.manage & SPS_BAM_MGR_DEVICE_REMOTE) != 0 && (dev->props.manage & SPS_BAM_MGR_MULTI_EE) != 0 && dev->props.ee == 0 && MTIenabled) { /* * BAM global is owned by remote processor and local processor * must use MTI. Thus, force EE index to a non-zero value to * insure that EE zero globals can't be modified. */ SPS_ERR("sps:EE for satellite BAM must be set to non-zero.\n"); return SPS_ERROR; } /* * Enable MTI use (message triggered interrupt) * if local processor does not control the global BAM config * and this BAM supports MTIs. */ if ((dev->state & BAM_STATE_IRQ) != 0 && (dev->props.manage & SPS_BAM_MGR_DEVICE_REMOTE) != 0 && MTIenabled) { if (dev->props.irq_gen_addr == 0 || dev->props.irq_gen_addr == SPS_ADDR_INVALID) { SPS_ERR( "sps:MTI destination address not specified for BAM 0x%x\n", BAM_ID(dev)); return SPS_ERROR; } dev->state |= BAM_STATE_MTI; } if (num_pipes) { dev->props.num_pipes = num_pipes; SPS_DBG1("sps:BAM 0x%x number of pipes reported by hw: %d\n", BAM_ID(dev), dev->props.num_pipes); } /* Check EE index */ if (!MTIenabled && dev->props.ee >= SPS_BAM_NUM_EES) { SPS_ERR("sps:Invalid EE BAM 0x%x: %d\n", BAM_ID(dev), dev->props.ee); return SPS_ERROR; } /* * Process EE configuration parameters, * if specified in the properties */ if (!MTIenabled && dev->props.sec_config == SPS_BAM_SEC_DO_CONFIG) { struct sps_bam_sec_config_props *p_sec = dev->props.p_sec_config_props; if (p_sec == NULL) { SPS_ERR( "sps:EE config table is not specified for BAM 0x%x\n", BAM_ID(dev)); return SPS_ERROR; } /* * Set restricted pipes based on the pipes assigned to local EE */ dev->props.restricted_pipes = ~p_sec->ees[dev->props.ee].pipe_mask; /* * If local processor manages the BAM, perform the EE * configuration */ if ((dev->props.manage & SPS_BAM_MGR_DEVICE_REMOTE) == 0) { u32 ee; u32 pipe_mask; int n, i; /* * Verify that there are no overlapping pipe * assignments */ for (n = 0; n < SPS_BAM_NUM_EES - 1; n++) { for (i = n + 1; i < SPS_BAM_NUM_EES; i++) { if ((p_sec->ees[n].pipe_mask & p_sec->ees[i].pipe_mask) != 0) { SPS_ERR( "sps:Overlapping pipe assignments for BAM 0x%x: EEs %d and %d\n", BAM_ID(dev), n, i); return SPS_ERROR; } } } for (ee = 0; ee < SPS_BAM_NUM_EES; ee++) { /* * MSbit specifies EE for the global (top-level) * BAM interrupt */ pipe_mask = p_sec->ees[ee].pipe_mask; if (ee == dev->props.ee) pipe_mask |= (1UL << 31); else pipe_mask &= ~(1UL << 31); bam_security_init(dev->base, ee, p_sec->ees[ee].vmid, pipe_mask); } } } /* * If local processor manages the BAM and the BAM supports MTIs * but does not support multiple EEs, set all restricted pipes * to MTI mode. */ if ((dev->props.manage & SPS_BAM_MGR_DEVICE_REMOTE) == 0 && MTIenabled) { u32 pipe_index; u32 pipe_mask; for (pipe_index = 0, pipe_mask = 1; pipe_index < dev->props.num_pipes; pipe_index++, pipe_mask <<= 1) { if ((pipe_mask & dev->props.restricted_pipes) == 0) continue; /* This is a local pipe */ /* * Enable MTI with destination address of zero * (and source mask zero). Pipe is in reset, * so no interrupt will be generated. */ bam_pipe_satellite_mti(dev->base, pipe_index, 0, dev->props.ee); } } dev->state |= BAM_STATE_ENABLED; if (!dev->props.constrained_logging || (dev->props.constrained_logging && dev->props.logging_number)) { if (dev->props.logging_number > 0) dev->props.logging_number--; SPS_INFO( "sps:BAM 0x%x (va:0x%x) enabled: ver:0x%x, number of pipes:%d\n", BAM_ID(dev), (u32) dev->base, dev->version, dev->props.num_pipes); } else SPS_DBG2( "sps:BAM 0x%x (va:0x%x) enabled: ver:0x%x, number of pipes:%d\n", BAM_ID(dev), (u32) dev->base, dev->version, dev->props.num_pipes); return 0; } /** * BAM device disable * */ int sps_bam_disable(struct sps_bam *dev) { if ((dev->state & BAM_STATE_ENABLED) == 0) return 0; /* Is there any access to this BAM? */ if ((dev->props.manage & SPS_BAM_MGR_ACCESS_MASK) == SPS_BAM_MGR_NONE) { SPS_ERR("sps:No local access to BAM 0x%x\n", BAM_ID(dev)); return SPS_ERROR; } /* Is this BAM controlled by the local processor? */ if ((dev->props.manage & SPS_BAM_MGR_DEVICE_REMOTE)) { /* No, so just mark it disabled */ dev->state &= ~BAM_STATE_ENABLED; return 0; } /* Disable BAM (interrupts) */ if ((dev->state & BAM_STATE_IRQ)) { bam_exit(dev->base, dev->props.ee); /* Deregister BAM ISR */ if ((dev->state & BAM_STATE_IRQ)) if (dev->props.irq > 0) free_irq(dev->props.irq, dev); dev->state &= ~BAM_STATE_IRQ; } dev->state &= ~BAM_STATE_ENABLED; SPS_DBG2("sps:BAM 0x%x disabled\n", BAM_ID(dev)); return 0; } /** * BAM device initialization */ int sps_bam_device_init(struct sps_bam *dev) { if (dev->props.virt_addr == NULL) { SPS_ERR("sps:NULL BAM virtual address\n"); return SPS_ERROR; } dev->base = (void *) dev->props.virt_addr; if (dev->props.num_pipes == 0) { /* Assume max number of pipes until BAM registers can be read */ dev->props.num_pipes = BAM_MAX_PIPES; SPS_DBG2("sps:BAM 0x%x: assuming max number of pipes: %d\n", BAM_ID(dev), dev->props.num_pipes); } /* Init BAM state data */ dev->state = 0; dev->pipe_active_mask = 0; dev->pipe_remote_mask = 0; INIT_LIST_HEAD(&dev->pipes_q); spin_lock_init(&dev->isr_lock); spin_lock_init(&dev->connection_lock); if ((dev->props.options & SPS_BAM_OPT_ENABLE_AT_BOOT)) if (sps_bam_enable(dev)) { SPS_ERR("sps:Fail to enable bam device\n"); return SPS_ERROR; } SPS_DBG2("sps:BAM device: phys 0x%x IRQ %d\n", BAM_ID(dev), dev->props.irq); return 0; } /** * BAM device de-initialization * */ int sps_bam_device_de_init(struct sps_bam *dev) { int result; SPS_DBG2("sps:BAM device DEINIT: phys 0x%x IRQ %d\n", BAM_ID(dev), dev->props.irq); result = sps_bam_disable(dev); return result; } /** * BAM device reset * */ int sps_bam_reset(struct sps_bam *dev) { struct sps_pipe *pipe; u32 pipe_index; int result; SPS_DBG2("sps:BAM device RESET: phys 0x%x IRQ %d\n", BAM_ID(dev), dev->props.irq); /* If BAM is enabled, then disable */ result = 0; if ((dev->state & BAM_STATE_ENABLED)) { /* Verify that no pipes are currently allocated */ for (pipe_index = 0; pipe_index < dev->props.num_pipes; pipe_index++) { pipe = dev->pipes[pipe_index]; if (BAM_PIPE_IS_ASSIGNED(pipe)) { SPS_ERR( "sps:BAM device 0x%x RESET failed: pipe %d in use\n", BAM_ID(dev), pipe_index); result = SPS_ERROR; break; } } if (result == 0) result = sps_bam_disable(dev); } /* BAM will be reset as part of the enable process */ if (result == 0) result = sps_bam_enable(dev); return result; } /** * Clear the BAM pipe state struct * * This function clears the BAM pipe state struct. * * @pipe - pointer to client pipe struct * */ static void pipe_clear(struct sps_pipe *pipe) { INIT_LIST_HEAD(&pipe->list); pipe->state = 0; pipe->pipe_index = SPS_BAM_PIPE_INVALID; pipe->pipe_index_mask = 0; pipe->irq_mask = 0; pipe->mode = -1; pipe->num_descs = 0; pipe->desc_size = 0; memset(&pipe->sys, 0, sizeof(pipe->sys)); INIT_LIST_HEAD(&pipe->sys.events_q); } /** * Allocate a BAM pipe * */ u32 sps_bam_pipe_alloc(struct sps_bam *dev, u32 pipe_index) { u32 pipe_mask; if (pipe_index == SPS_BAM_PIPE_INVALID) { /* Allocate a pipe from the BAM */ if ((dev->props.manage & SPS_BAM_MGR_PIPE_NO_ALLOC)) { SPS_ERR( "sps:Restricted from allocating pipes on BAM 0x%x\n", BAM_ID(dev)); return SPS_BAM_PIPE_INVALID; } for (pipe_index = 0, pipe_mask = 1; pipe_index < dev->props.num_pipes; pipe_index++, pipe_mask <<= 1) { if ((pipe_mask & dev->props.restricted_pipes)) continue; /* This is a restricted pipe */ if (dev->pipes[pipe_index] == NULL) break; /* Found an available pipe */ } if (pipe_index >= dev->props.num_pipes) { SPS_ERR("sps:Fail to allocate pipe on BAM 0x%x\n", BAM_ID(dev)); return SPS_BAM_PIPE_INVALID; } } else { /* Check that client-specified pipe is available */ if (pipe_index >= dev->props.num_pipes) { SPS_ERR( "sps:Invalid pipe %d for allocate on BAM 0x%x\n", pipe_index, BAM_ID(dev)); return SPS_BAM_PIPE_INVALID; } if ((dev->props.restricted_pipes & (1UL << pipe_index))) { SPS_ERR("sps:BAM 0x%x pipe %d is not local\n", BAM_ID(dev), pipe_index); return SPS_BAM_PIPE_INVALID; } if (dev->pipes[pipe_index] != NULL) { SPS_ERR("sps:Pipe %d already allocated on BAM 0x%x\n", pipe_index, BAM_ID(dev)); return SPS_BAM_PIPE_INVALID; } } /* Mark pipe as allocated */ dev->pipes[pipe_index] = BAM_PIPE_UNASSIGNED; return pipe_index; } /** * Free a BAM pipe * */ void sps_bam_pipe_free(struct sps_bam *dev, u32 pipe_index) { struct sps_pipe *pipe; if (pipe_index >= dev->props.num_pipes) { SPS_ERR("sps:Invalid BAM 0x%x pipe: %d\n", BAM_ID(dev), pipe_index); return; } /* Get the client pipe struct and mark the pipe free */ pipe = dev->pipes[pipe_index]; dev->pipes[pipe_index] = NULL; /* Is the pipe currently allocated? */ if (pipe == NULL) { SPS_ERR("sps:Attempt to free unallocated pipe %d on BAM 0x%x\n", pipe_index, BAM_ID(dev)); return; } if (pipe == BAM_PIPE_UNASSIGNED) return; /* Never assigned, so no work to do */ /* Return pending items to appropriate pools */ if (!list_empty(&pipe->sys.events_q)) { struct sps_q_event *sps_event; SPS_ERR("sps:Disconnect BAM 0x%x pipe %d with events pending\n", BAM_ID(dev), pipe_index); sps_event = list_entry((&pipe->sys.events_q)->next, typeof(*sps_event), list); while (&sps_event->list != (&pipe->sys.events_q)) { struct sps_q_event *sps_event_delete = sps_event; list_del(&sps_event->list); sps_event = list_entry(sps_event->list.next, typeof(*sps_event), list); kfree(sps_event_delete); } } /* Clear the BAM pipe state struct */ pipe_clear(pipe); } /** * Establish BAM pipe connection * */ int sps_bam_pipe_connect(struct sps_pipe *bam_pipe, const struct sps_bam_connect_param *params) { struct bam_pipe_parameters hw_params; struct sps_bam *dev; const struct sps_connection *map = bam_pipe->map; const struct sps_conn_end_pt *map_pipe; const struct sps_conn_end_pt *other_pipe; void *desc_buf = NULL; u32 pipe_index; int result; /* Clear the client pipe state and hw init struct */ pipe_clear(bam_pipe); memset(&hw_params, 0, sizeof(hw_params)); /* Initialize the BAM state struct */ bam_pipe->mode = params->mode; /* Set pipe streaming mode */ if ((params->options & SPS_O_STREAMING) == 0) hw_params.stream_mode = BAM_STREAM_MODE_DISABLE; else hw_params.stream_mode = BAM_STREAM_MODE_ENABLE; /* Determine which end point to connect */ if (bam_pipe->mode == SPS_MODE_SRC) { map_pipe = &map->src; other_pipe = &map->dest; hw_params.dir = BAM_PIPE_PRODUCER; } else { map_pipe = &map->dest; other_pipe = &map->src; hw_params.dir = BAM_PIPE_CONSUMER; } /* Process map parameters */ dev = map_pipe->bam; pipe_index = map_pipe->pipe_index; if (pipe_index >= dev->props.num_pipes) { SPS_ERR("sps:Invalid BAM 0x%x pipe: %d\n", BAM_ID(dev), pipe_index); return SPS_ERROR; } hw_params.event_threshold = (u16) map_pipe->event_threshold; hw_params.ee = dev->props.ee; hw_params.lock_group = map_pipe->lock_group; /* Verify that control of this pipe is allowed */ if ((dev->props.manage & SPS_BAM_MGR_PIPE_NO_CTRL) || (dev->props.restricted_pipes & (1UL << pipe_index))) { SPS_ERR("sps:BAM 0x%x pipe %d is not local\n", BAM_ID(dev), pipe_index); return SPS_ERROR; } /* Control without configuration permission is not supported yet */ if ((dev->props.manage & SPS_BAM_MGR_PIPE_NO_CONFIG)) { SPS_ERR("sps:BAM 0x%x pipe %d remote config is not supported\n", BAM_ID(dev), pipe_index); return SPS_ERROR; } /* Determine operational mode */ if (other_pipe->bam != NULL) { /* BAM-to-BAM mode */ bam_pipe->state |= BAM_STATE_BAM2BAM; hw_params.mode = BAM_PIPE_MODE_BAM2BAM; hw_params.peer_phys_addr = ((struct sps_bam *) (other_pipe->bam))->props.phys_addr; hw_params.peer_pipe = other_pipe->pipe_index; /* Verify FIFO buffers are allocated for BAM-to-BAM pipes */ if (map->desc.phys_base == SPS_ADDR_INVALID || map->data.phys_base == SPS_ADDR_INVALID || map->desc.size == 0 || map->data.size == 0) { SPS_ERR( "sps:FIFO buffers are not allocated for BAM 0x%x pipe %d.\n", BAM_ID(dev), pipe_index); return SPS_ERROR; } hw_params.data_base = map->data.phys_base; hw_params.data_size = map->data.size; /* Clear the data FIFO for debug */ if (map->data.base != NULL && bam_pipe->mode == SPS_MODE_SRC) memset(map->data.base, 0, hw_params.data_size); /* set NWD bit for BAM2BAM producer pipe */ if (bam_pipe->mode == SPS_MODE_SRC) { if ((params->options & SPS_O_WRITE_NWD) == 0) hw_params.write_nwd = BAM_WRITE_NWD_DISABLE; else hw_params.write_nwd = BAM_WRITE_NWD_ENABLE; } } else { /* System mode */ hw_params.mode = BAM_PIPE_MODE_SYSTEM; bam_pipe->sys.desc_buf = map->desc.base; bam_pipe->sys.desc_offset = 0; bam_pipe->sys.acked_offset = 0; } /* Initialize the client pipe state */ bam_pipe->pipe_index = pipe_index; bam_pipe->pipe_index_mask = 1UL << pipe_index; /* Get virtual address for descriptor FIFO */ if (map->desc.phys_base != SPS_ADDR_INVALID) { if (map->desc.size < (2 * sizeof(struct sps_iovec))) { SPS_ERR( "sps:Invalid descriptor FIFO size for BAM 0x%x pipe %d: %d\n", BAM_ID(dev), pipe_index, map->desc.size); return SPS_ERROR; } desc_buf = map->desc.base; /* * Note that descriptor base and size will be left zero from * the memset() above if the physical address was invalid. * This allows a satellite driver to set the FIFO as * local memory for system mode. */ hw_params.desc_base = map->desc.phys_base; hw_params.desc_size = map->desc.size; } /* Configure the descriptor FIFO for both operational modes */ if (desc_buf != NULL) if (bam_pipe->mode == SPS_MODE_SRC || hw_params.mode == BAM_PIPE_MODE_SYSTEM) memset(desc_buf, 0, hw_params.desc_size); bam_pipe->desc_size = hw_params.desc_size; bam_pipe->num_descs = bam_pipe->desc_size / sizeof(struct sps_iovec); result = SPS_ERROR; /* Insure that the BAM is enabled */ if ((dev->state & BAM_STATE_ENABLED) == 0) if (sps_bam_enable(dev)) goto exit_init_err; /* Check pipe allocation */ if (dev->pipes[pipe_index] != BAM_PIPE_UNASSIGNED) { SPS_ERR("sps:Invalid pipe %d on BAM 0x%x for connect\n", pipe_index, BAM_ID(dev)); return SPS_ERROR; } if (bam_pipe_is_enabled(dev->base, pipe_index)) { if (params->options & SPS_O_NO_DISABLE) SPS_DBG("sps:BAM 0x%x pipe %d is already enabled\n", BAM_ID(dev), pipe_index); else { SPS_ERR("sps:BAM 0x%x pipe %d sharing violation\n", BAM_ID(dev), pipe_index); return SPS_ERROR; } } if (bam_pipe_init(dev->base, pipe_index, &hw_params, dev->props.ee)) { SPS_ERR("sps:BAM 0x%x pipe %d init error\n", BAM_ID(dev), pipe_index); goto exit_err; } /* Assign pipe to client */ dev->pipes[pipe_index] = bam_pipe; /* Process configuration parameters */ if (params->options != 0 || (bam_pipe->state & BAM_STATE_BAM2BAM) == 0) { /* Process init-time only parameters */ u32 irq_gen_addr; /* Set interrupt mode */ irq_gen_addr = SPS_ADDR_INVALID; if ((params->options & SPS_O_IRQ_MTI)) /* Client has directly specified the MTI address */ irq_gen_addr = params->irq_gen_addr; else if ((dev->state & BAM_STATE_MTI)) /* This BAM has MTI use enabled */ irq_gen_addr = dev->props.irq_gen_addr; if (irq_gen_addr != SPS_ADDR_INVALID) { /* * No checks - assume BAM is already setup for * MTI generation, * or the pipe will be set to satellite control. */ bam_pipe->state |= BAM_STATE_MTI; bam_pipe->irq_gen_addr = irq_gen_addr; } /* Process runtime parameters */ if (sps_bam_pipe_set_params(dev, pipe_index, params->options)) { dev->pipes[pipe_index] = BAM_PIPE_UNASSIGNED; goto exit_err; } } /* Indicate initialization is complete */ dev->pipes[pipe_index] = bam_pipe; dev->pipe_active_mask |= 1UL << pipe_index; list_add_tail(&bam_pipe->list, &dev->pipes_q); bam_pipe->state |= BAM_STATE_INIT; result = 0; exit_err: if (result) { if (params->options & SPS_O_NO_DISABLE) SPS_DBG("sps:BAM 0x%x pipe %d connection exits\n", BAM_ID(dev), pipe_index); else bam_pipe_exit(dev->base, pipe_index, dev->props.ee); } exit_init_err: if (result) { /* Clear the client pipe state */ pipe_clear(bam_pipe); } return result; } /** * Disconnect a BAM pipe connection * */ int sps_bam_pipe_disconnect(struct sps_bam *dev, u32 pipe_index) { struct sps_pipe *pipe; int result; if (pipe_index >= dev->props.num_pipes) { SPS_ERR("sps:Invalid BAM 0x%x pipe: %d\n", BAM_ID(dev), pipe_index); return SPS_ERROR; } /* Deallocate and reset the BAM pipe */ pipe = dev->pipes[pipe_index]; if (BAM_PIPE_IS_ASSIGNED(pipe)) { if ((dev->pipe_active_mask & (1UL << pipe_index))) { list_del(&pipe->list); dev->pipe_active_mask &= ~(1UL << pipe_index); } dev->pipe_remote_mask &= ~(1UL << pipe_index); if (pipe->connect.options & SPS_O_NO_DISABLE) SPS_DBG("sps:BAM 0x%x pipe %d exits\n", BAM_ID(dev), pipe_index); else bam_pipe_exit(dev->base, pipe_index, dev->props.ee); if (pipe->sys.desc_cache != NULL) { u32 size = pipe->num_descs * sizeof(void *); if (pipe->desc_size + size <= PAGE_SIZE) kfree(pipe->sys.desc_cache); else vfree(pipe->sys.desc_cache); pipe->sys.desc_cache = NULL; } dev->pipes[pipe_index] = BAM_PIPE_UNASSIGNED; pipe_clear(pipe); result = 0; } else { result = SPS_ERROR; } if (result) SPS_ERR("sps:BAM 0x%x pipe %d already disconnected\n", BAM_ID(dev), pipe_index); return result; } /** * Set BAM pipe interrupt enable state * * This function sets the interrupt enable state for a BAM pipe. * * @dev - pointer to BAM device descriptor * * @pipe_index - pipe index * * @poll - true if SPS_O_POLL is set, false otherwise * */ static void pipe_set_irq(struct sps_bam *dev, u32 pipe_index, u32 poll) { struct sps_pipe *pipe = dev->pipes[pipe_index]; enum bam_enable irq_enable; if (poll == 0 && pipe->irq_mask != 0 && (dev->state & BAM_STATE_IRQ)) { if ((pipe->state & BAM_STATE_BAM2BAM) != 0 && (pipe->state & BAM_STATE_IRQ) == 0) { /* * If enabling the interrupt for a BAM-to-BAM pipe, * clear the existing interrupt status */ (void)bam_pipe_get_and_clear_irq_status(dev->base, pipe_index); } pipe->state |= BAM_STATE_IRQ; irq_enable = BAM_ENABLE; pipe->polled = false; } else { pipe->state &= ~BAM_STATE_IRQ; irq_enable = BAM_DISABLE; pipe->polled = true; if (poll == 0 && pipe->irq_mask) SPS_DBG2("sps:BAM 0x%x pipe %d forced to use polling\n", BAM_ID(dev), pipe_index); } if ((pipe->state & BAM_STATE_MTI) == 0) bam_pipe_set_irq(dev->base, pipe_index, irq_enable, pipe->irq_mask, dev->props.ee); else bam_pipe_set_mti(dev->base, pipe_index, irq_enable, pipe->irq_mask, pipe->irq_gen_addr); } /** * Set BAM pipe parameters * */ int sps_bam_pipe_set_params(struct sps_bam *dev, u32 pipe_index, u32 options) { struct sps_pipe *pipe = dev->pipes[pipe_index]; u32 mask; int wake_up_is_one_shot; int no_queue; int ack_xfers; u32 size; int n; /* Capture some options */ wake_up_is_one_shot = ((options & SPS_O_WAKEUP_IS_ONESHOT)); no_queue = ((options & SPS_O_NO_Q)); ack_xfers = ((options & SPS_O_ACK_TRANSFERS)); pipe->hybrid = options & SPS_O_HYBRID; /* Create interrupt source mask */ mask = 0; for (n = 0; n < ARRAY_SIZE(opt_event_table); n++) { /* Is client registering for this event? */ if ((options & opt_event_table[n].option) == 0) continue; /* No */ mask |= opt_event_table[n].pipe_irq; } #ifdef SPS_BAM_STATISTICS /* Is an illegal mode change specified? */ if (pipe->sys.desc_wr_count > 0 && (no_queue != pipe->sys.no_queue || ack_xfers != pipe->sys.ack_xfers)) { SPS_ERR( "sps:Queue/ack mode change after transfer: BAM 0x%x pipe %d opt 0x%x\n", BAM_ID(dev), pipe_index, options); return SPS_ERROR; } #endif /* SPS_BAM_STATISTICS */ /* Is client setting invalid options for a BAM-to-BAM connection? */ if ((pipe->state & BAM_STATE_BAM2BAM) && (options & BAM2BAM_O_INVALID)) { SPS_ERR( "sps:Invalid option for BAM-to-BAM: BAM 0x%x pipe %d opt 0x%x\n", BAM_ID(dev), pipe_index, options); return SPS_ERROR; } /* Allocate descriptor FIFO cache if NO_Q option is disabled */ if (!no_queue && pipe->sys.desc_cache == NULL && pipe->num_descs > 0 && (pipe->state & BAM_STATE_BAM2BAM) == 0) { /* Allocate both descriptor cache and user pointer array */ size = pipe->num_descs * sizeof(void *); if (pipe->desc_size + size <= PAGE_SIZE) pipe->sys.desc_cache = kzalloc(pipe->desc_size + size, GFP_KERNEL); else { pipe->sys.desc_cache = vmalloc(pipe->desc_size + size); if (pipe->sys.desc_cache == NULL) { SPS_ERR( "sps:No memory for pipe %d of BAM 0x%x\n", pipe_index, BAM_ID(dev)); return -ENOMEM; } memset(pipe->sys.desc_cache, 0, pipe->desc_size + size); } if (pipe->sys.desc_cache == NULL) { /*** MUST BE LAST POINT OF FAILURE (see below) *****/ SPS_ERR("sps:Desc cache error: BAM 0x%x pipe %d: %d\n", BAM_ID(dev), pipe_index, pipe->desc_size + size); return SPS_ERROR; } pipe->sys.user_ptrs = (void **)(pipe->sys.desc_cache + pipe->desc_size); pipe->sys.cache_offset = pipe->sys.acked_offset; } /* * No failures beyond this point. Note that malloc() is last point of * failure, so no free() handling is needed. */ /* Enable/disable the pipe's interrupt sources */ pipe->irq_mask = mask; pipe_set_irq(dev, pipe_index, (options & SPS_O_POLL)); /* Store software feature enables */ pipe->wake_up_is_one_shot = wake_up_is_one_shot; pipe->sys.no_queue = no_queue; pipe->sys.ack_xfers = ack_xfers; return 0; } /** * Enable a BAM pipe * */ int sps_bam_pipe_enable(struct sps_bam *dev, u32 pipe_index) { struct sps_pipe *pipe = dev->pipes[pipe_index]; /* Enable the BAM pipe */ bam_pipe_enable(dev->base, pipe_index); pipe->state |= BAM_STATE_ENABLED; return 0; } /** * Disable a BAM pipe * */ int sps_bam_pipe_disable(struct sps_bam *dev, u32 pipe_index) { struct sps_pipe *pipe = dev->pipes[pipe_index]; /* Disable the BAM pipe */ if (pipe->connect.options & SPS_O_NO_DISABLE) SPS_DBG("sps:BAM 0x%x pipe %d enters disable state\n", BAM_ID(dev), pipe_index); else bam_pipe_disable(dev->base, pipe_index); pipe->state &= ~BAM_STATE_ENABLED; return 0; } /** * Register an event for a BAM pipe * */ int sps_bam_pipe_reg_event(struct sps_bam *dev, u32 pipe_index, struct sps_register_event *reg) { struct sps_pipe *pipe = dev->pipes[pipe_index]; struct sps_bam_event_reg *event_reg; int n; if (pipe->sys.no_queue && reg->xfer_done != NULL && reg->mode != SPS_TRIGGER_CALLBACK) { SPS_ERR( "sps:Only callback events support for NO_Q: BAM 0x%x pipe %d mode %d\n", BAM_ID(dev), pipe_index, reg->mode); return SPS_ERROR; } for (n = 0; n < ARRAY_SIZE(opt_event_table); n++) { int index; /* Is client registering for this event? */ if ((reg->options & opt_event_table[n].option) == 0) continue; /* No */ index = SPS_EVENT_INDEX(opt_event_table[n].event_id); if (index < 0) SPS_ERR( "sps:Negative event index: BAM 0x%x pipe %d mode %d\n", BAM_ID(dev), pipe_index, reg->mode); else { event_reg = &pipe->sys.event_regs[index]; event_reg->xfer_done = reg->xfer_done; event_reg->callback = reg->callback; event_reg->mode = reg->mode; event_reg->user = reg->user; } } return 0; } /** * Submit a transfer of a single buffer to a BAM pipe * */ int sps_bam_pipe_transfer_one(struct sps_bam *dev, u32 pipe_index, u32 addr, u32 size, void *user, u32 flags) { struct sps_pipe *pipe = dev->pipes[pipe_index]; struct sps_iovec *desc; struct sps_iovec iovec; u32 next_write; static int show_recom; /* Is this a BAM-to-BAM or satellite connection? */ if ((pipe->state & (BAM_STATE_BAM2BAM | BAM_STATE_REMOTE))) { SPS_ERR("sps:Transfer on BAM-to-BAM: BAM 0x%x pipe %d\n", BAM_ID(dev), pipe_index); return SPS_ERROR; } /* * Client identifier (user pointer) is not supported for * SPS_O_NO_Q option. */ if (pipe->sys.no_queue && user != NULL) { SPS_ERR("sps:User pointer arg non-NULL: BAM 0x%x pipe %d\n", BAM_ID(dev), pipe_index); return SPS_ERROR; } /* Determine if descriptor can be queued */ next_write = pipe->sys.desc_offset + sizeof(struct sps_iovec); if (next_write >= pipe->desc_size) next_write = 0; if (next_write == pipe->sys.acked_offset) { /* * If pipe is polled and client is not ACK'ing descriptors, * perform polling operation so that any outstanding ACKs * can occur. */ if (!pipe->sys.ack_xfers && pipe->polled) { pipe_handler_eot(dev, pipe); if (next_write == pipe->sys.acked_offset) { if (!show_recom) { show_recom = true; SPS_ERR( "sps:Client of BAM 0x%x pipe %d is recommended to have flow control\n", BAM_ID(dev), pipe_index); } SPS_DBG2( "sps:Descriptor FIFO is full for BAM 0x%x pipe %d after pipe_handler_eot\n", BAM_ID(dev), pipe_index); return SPS_ERROR; } } else { if (!show_recom) { show_recom = true; SPS_ERR( "sps:Client of BAM 0x%x pipe %d is recommended to have flow control.\n", BAM_ID(dev), pipe_index); } SPS_DBG2( "sps:Descriptor FIFO is full for BAM 0x%x pipe %d\n", BAM_ID(dev), pipe_index); return SPS_ERROR; } } /* Create descriptor */ if (!pipe->sys.no_queue) desc = (struct sps_iovec *) (pipe->sys.desc_cache + pipe->sys.desc_offset); else desc = &iovec; desc->addr = addr; desc->size = size; if ((flags & SPS_IOVEC_FLAG_DEFAULT) == 0) { desc->flags = flags & BAM_IOVEC_FLAG_MASK; } else { if (pipe->mode == SPS_MODE_SRC) desc->flags = SPS_IOVEC_FLAG_INT; else desc->flags = SPS_IOVEC_FLAG_INT | SPS_IOVEC_FLAG_EOT; } #ifdef SPS_BAM_STATISTICS if ((flags & SPS_IOVEC_FLAG_INT)) pipe->sys.int_flags++; if ((flags & SPS_IOVEC_FLAG_EOT)) pipe->sys.eot_flags++; #endif /* SPS_BAM_STATISTICS */ /* Update hardware descriptor FIFO - should result in burst */ *((struct sps_iovec *) (pipe->sys.desc_buf + pipe->sys.desc_offset)) = *desc; /* Record user pointer value */ if (!pipe->sys.no_queue) { u32 index = pipe->sys.desc_offset / sizeof(struct sps_iovec); pipe->sys.user_ptrs[index] = user; #ifdef SPS_BAM_STATISTICS if (user != NULL) pipe->sys.user_ptrs_count++; #endif /* SPS_BAM_STATISTICS */ } /* Update descriptor ACK offset */ pipe->sys.desc_offset = next_write; #ifdef SPS_BAM_STATISTICS /* Update statistics */ pipe->sys.desc_wr_count++; #endif /* SPS_BAM_STATISTICS */ /* Notify pipe */ if ((flags & SPS_IOVEC_FLAG_NO_SUBMIT) == 0) { wmb(); /* Memory Barrier */ bam_pipe_set_desc_write_offset(dev->base, pipe_index, next_write); } return 0; } /** * Submit a transfer to a BAM pipe * */ int sps_bam_pipe_transfer(struct sps_bam *dev, u32 pipe_index, struct sps_transfer *transfer) { struct sps_iovec *iovec; u32 count; u32 flags; void *user; int n; int result; if (transfer->iovec_count == 0) { SPS_ERR("sps:iovec count zero: BAM 0x%x pipe %d\n", BAM_ID(dev), pipe_index); return SPS_ERROR; } sps_bam_get_free_count(dev, pipe_index, &count); if (count < transfer->iovec_count) { SPS_ERR("sps:Insufficient free desc: BAM 0x%x pipe %d: %d\n", BAM_ID(dev), pipe_index, count); return SPS_ERROR; } user = NULL; /* NULL for all except last descriptor */ for (n = (int)transfer->iovec_count - 1, iovec = transfer->iovec; n >= 0; n--, iovec++) { if (n > 0) { /* This is *not* the last descriptor */ flags = iovec->flags | SPS_IOVEC_FLAG_NO_SUBMIT; } else { /* This *is* the last descriptor */ flags = iovec->flags; user = transfer->user; } result = sps_bam_pipe_transfer_one(dev, pipe_index, iovec->addr, iovec->size, user, flags); if (result) return SPS_ERROR; } return 0; } /** * Allocate an event tracking struct * * This function allocates an event tracking struct. * * @pipe - pointer to pipe state * * @event_reg - pointer to event registration * * @return - pointer to event notification struct, or NULL * */ static struct sps_q_event *alloc_event(struct sps_pipe *pipe, struct sps_bam_event_reg *event_reg) { struct sps_q_event *event; /* A callback event object is registered, so trigger with payload */ event = &pipe->sys.event; memset(event, 0, sizeof(*event)); return event; } /** * Trigger an event notification * * This function triggers an event notification. * * @dev - pointer to BAM device descriptor * * @pipe - pointer to pipe state * * @event_reg - pointer to event registration * * @sps_event - pointer to event struct * */ static void trigger_event(struct sps_bam *dev, struct sps_pipe *pipe, struct sps_bam_event_reg *event_reg, struct sps_q_event *sps_event) { if (sps_event == NULL) { SPS_DBG("sps:trigger_event.sps_event is NULL.\n"); return; } if (event_reg->xfer_done) { complete(event_reg->xfer_done); SPS_DBG("sps:trigger_event.done=%d.\n", event_reg->xfer_done->done); } if (event_reg->callback) { SPS_DBG("sps:trigger_event.using callback.\n"); event_reg->callback(&sps_event->notify); } } /** * Handle a BAM pipe's generic interrupt sources * * This function creates the event notification for a BAM pipe's * generic interrupt sources. The caller of this function must lock the BAM * device's mutex. * * @dev - pointer to BAM device descriptor * * @pipe - pointer to pipe state * * @event_id - event identifier enum * */ static void pipe_handler_generic(struct sps_bam *dev, struct sps_pipe *pipe, enum sps_event event_id) { struct sps_bam_event_reg *event_reg; struct sps_q_event *sps_event; int index; index = SPS_EVENT_INDEX(event_id); if (index < 0 || index >= SPS_EVENT_INDEX(SPS_EVENT_MAX)) return; event_reg = &pipe->sys.event_regs[index]; sps_event = alloc_event(pipe, event_reg); if (sps_event != NULL) { sps_event->notify.event_id = event_id; sps_event->notify.user = event_reg->user; trigger_event(dev, pipe, event_reg, sps_event); } } /** * Handle a BAM pipe's WAKEUP interrupt sources * * This function creates the event notification for a BAM pipe's * WAKEUP interrupt source. The caller of this function must lock the BAM * device's mutex. * * @dev - pointer to BAM device descriptor * * @pipe - pointer to pipe state * */ static void pipe_handler_wakeup(struct sps_bam *dev, struct sps_pipe *pipe) { struct sps_bam_event_reg *event_reg; struct sps_q_event *event; u32 pipe_index = pipe->pipe_index; if (pipe->wake_up_is_one_shot) { /* Disable the pipe WAKEUP interrupt source */ pipe->irq_mask &= ~BAM_PIPE_IRQ_WAKE; pipe_set_irq(dev, pipe_index, pipe->polled); } event_reg = &pipe->sys.event_regs[SPS_EVENT_INDEX(SPS_EVENT_WAKEUP)]; event = alloc_event(pipe, event_reg); if (event != NULL) { event->notify.event_id = SPS_EVENT_WAKEUP; event->notify.user = event_reg->user; trigger_event(dev, pipe, event_reg, event); } } /** * Handle a BAM pipe's EOT/INT interrupt sources * * This function creates the event notification for a BAM pipe's EOT interrupt * source. The caller of this function must lock the BAM device's mutex. * * @dev - pointer to BAM device descriptor * * @pipe - pointer to pipe state * */ static void pipe_handler_eot(struct sps_bam *dev, struct sps_pipe *pipe) { struct sps_bam_event_reg *event_reg; struct sps_q_event *event; struct sps_iovec *desc; struct sps_iovec *cache; void **user; u32 *update_offset; u32 pipe_index = pipe->pipe_index; u32 offset; u32 end_offset; enum sps_event event_id; u32 flags; u32 enabled; int producer = (pipe->mode == SPS_MODE_SRC); if (pipe->sys.handler_eot) /* * This can happen if the pipe is configured for polling * (IRQ disabled) and callback event generation. * The client may perform a get_iovec() inside the callback. */ return; pipe->sys.handler_eot = true; /* Get offset of last descriptor completed by the pipe */ end_offset = bam_pipe_get_desc_read_offset(dev->base, pipe_index); /* If no queue, then do not generate any events */ if (pipe->sys.no_queue) { if (!pipe->sys.ack_xfers) { /* Client is not ACK'ing transfers, so do it now */ pipe->sys.acked_offset = end_offset; } pipe->sys.handler_eot = false; return; } /* * Get offset of last descriptor processed by software, * and update to the last descriptor completed by the pipe */ if (!pipe->sys.ack_xfers) { update_offset = &pipe->sys.acked_offset; offset = *update_offset; } else { update_offset = &pipe->sys.cache_offset; offset = *update_offset; } /* Are there any completed descriptors to process? */ if (offset == end_offset) { pipe->sys.handler_eot = false; return; } /* Determine enabled events */ enabled = 0; if ((pipe->irq_mask & SPS_O_EOT)) enabled |= SPS_IOVEC_FLAG_EOT; if ((pipe->irq_mask & SPS_O_DESC_DONE)) enabled |= SPS_IOVEC_FLAG_INT; /* * For producer pipe, update the cached descriptor byte count and flags. * For consumer pipe, the BAM does not update the descriptors, so just * use the cached copies. */ if (producer) { /* * Do copies in a tight loop to increase chance of * multi-descriptor burst accesses on the bus */ struct sps_iovec *desc_end; /* Set starting point for copy */ desc = (struct sps_iovec *) (pipe->sys.desc_buf + offset); cache = (struct sps_iovec *) (pipe->sys.desc_cache + offset); /* Fetch all completed descriptors to end of FIFO (wrap) */ if (end_offset < offset) { desc_end = (struct sps_iovec *) (pipe->sys.desc_buf + pipe->desc_size); while (desc < desc_end) *cache++ = *desc++; desc = (void *)pipe->sys.desc_buf; cache = (void *)pipe->sys.desc_cache; } /* Fetch all remaining completed descriptors (no wrap) */ desc_end = (struct sps_iovec *) (pipe->sys.desc_buf + end_offset); while (desc < desc_end) *cache++ = *desc++; } /* Process all completed descriptors */ cache = (struct sps_iovec *) (pipe->sys.desc_cache + offset); user = &pipe->sys.user_ptrs[offset / sizeof(struct sps_iovec)]; for (;;) { /* * Increment offset to next descriptor and update pipe offset * so a client callback can fetch the I/O vector. */ offset += sizeof(struct sps_iovec); if (offset >= pipe->desc_size) /* Roll to start of descriptor FIFO */ offset = 0; *update_offset = offset; #ifdef SPS_BAM_STATISTICS pipe->sys.desc_rd_count++; #endif /* SPS_BAM_STATISTICS */ /* Did client request notification for this descriptor? */ flags = cache->flags & enabled; if (*user != NULL || flags) { int index; if ((flags & SPS_IOVEC_FLAG_EOT)) event_id = SPS_EVENT_EOT; else event_id = SPS_EVENT_DESC_DONE; index = SPS_EVENT_INDEX(event_id); event_reg = &pipe->sys.event_regs[index]; event = alloc_event(pipe, event_reg); if (event != NULL) { /* * Store the descriptor and user pointer * in the notification */ event->notify.data.transfer.iovec = *cache; event->notify.data.transfer.user = *user; event->notify.event_id = event_id; event->notify.user = event_reg->user; trigger_event(dev, pipe, event_reg, event); } #ifdef SPS_BAM_STATISTICS if (*user != NULL) pipe->sys.user_found++; #endif /* SPS_BAM_STATISTICS */ } /* Increment to next descriptor */ if (offset == end_offset) break; /* No more descriptors */ if (offset) { cache++; user++; } else { cache = (void *)pipe->sys.desc_cache; user = pipe->sys.user_ptrs; } } pipe->sys.handler_eot = false; } /** * Handle a BAM pipe's interrupt sources * * This function handles a BAM pipe's interrupt sources. * The caller of this function must lock the BAM device's mutex. * * @dev - pointer to BAM device descriptor * * @pipe_index - pipe index * * @return void * */ static void pipe_handler(struct sps_bam *dev, struct sps_pipe *pipe) { u32 pipe_index; u32 status; enum sps_event event_id; /* Get interrupt sources and ack all */ pipe_index = pipe->pipe_index; status = bam_pipe_get_and_clear_irq_status(dev->base, pipe_index); SPS_DBG("sps:pipe_handler.bam 0x%x.pipe %d.status=0x%x.\n", BAM_ID(dev), pipe_index, status); /* Check for enabled interrupt sources */ status &= pipe->irq_mask; if (status == 0) /* No enabled interrupt sources are active */ return; /* * Process the interrupt sources in order of frequency of occurrance. * Check for early exit opportunities. */ if ((status & (SPS_O_EOT | SPS_O_DESC_DONE)) && (pipe->state & BAM_STATE_BAM2BAM) == 0) { pipe_handler_eot(dev, pipe); if (pipe->sys.no_queue) { /* * EOT handler will not generate any event if there * is no queue, * so generate "empty" (no descriptor) event */ if ((status & SPS_O_EOT)) event_id = SPS_EVENT_EOT; else event_id = SPS_EVENT_DESC_DONE; pipe_handler_generic(dev, pipe, event_id); } status &= ~(SPS_O_EOT | SPS_O_DESC_DONE); if (status == 0) return; } if ((status & SPS_O_WAKEUP)) { pipe_handler_wakeup(dev, pipe); status &= ~SPS_O_WAKEUP; if (status == 0) return; } if ((status & SPS_O_INACTIVE)) { pipe_handler_generic(dev, pipe, SPS_EVENT_INACTIVE); status &= ~SPS_O_INACTIVE; if (status == 0) return; } if ((status & SPS_O_OUT_OF_DESC)) { pipe_handler_generic(dev, pipe, SPS_EVENT_OUT_OF_DESC); status &= ~SPS_O_OUT_OF_DESC; if (status == 0) return; } if ((status & SPS_EVENT_ERROR)) pipe_handler_generic(dev, pipe, SPS_EVENT_ERROR); } /** * Get a BAM pipe event * */ int sps_bam_pipe_get_event(struct sps_bam *dev, u32 pipe_index, struct sps_event_notify *notify) { struct sps_pipe *pipe = dev->pipes[pipe_index]; struct sps_q_event *event_queue; if (pipe->sys.no_queue) { SPS_ERR( "sps:Invalid connection for event: BAM 0x%x pipe %d context 0x%x\n", BAM_ID(dev), pipe_index, (u32) pipe); notify->event_id = SPS_EVENT_INVALID; return SPS_ERROR; } /* If pipe is polled, perform polling operation */ if (pipe->polled && (pipe->state & BAM_STATE_BAM2BAM) == 0) pipe_handler_eot(dev, pipe); /* Pull an event off the synchronous event queue */ if (list_empty(&pipe->sys.events_q)) { event_queue = NULL; SPS_DBG("sps:events_q of bam 0x%x is empty.\n", BAM_ID(dev)); } else { SPS_DBG("sps:events_q of bam 0x%x is not empty.\n", BAM_ID(dev)); event_queue = list_first_entry(&pipe->sys.events_q, struct sps_q_event, list); list_del(&event_queue->list); } /* Update client's event buffer */ if (event_queue == NULL) { /* No event queued, so set client's event to "invalid" */ notify->event_id = SPS_EVENT_INVALID; } else { /* * Copy event into client's buffer and return the event * to the pool */ *notify = event_queue->notify; kfree(event_queue); #ifdef SPS_BAM_STATISTICS pipe->sys.get_events++; #endif /* SPS_BAM_STATISTICS */ } return 0; } /** * Get processed I/O vector */ int sps_bam_pipe_get_iovec(struct sps_bam *dev, u32 pipe_index, struct sps_iovec *iovec) { struct sps_pipe *pipe = dev->pipes[pipe_index]; struct sps_iovec *desc; u32 read_offset; /* Is this a valid pipe configured for get_iovec use? */ if (!pipe->sys.ack_xfers || (pipe->state & BAM_STATE_BAM2BAM) != 0 || (pipe->state & BAM_STATE_REMOTE)) { return SPS_ERROR; } /* If pipe is polled and queue is enabled, perform polling operation */ if ((pipe->polled || pipe->hybrid) && !pipe->sys.no_queue) pipe_handler_eot(dev, pipe); /* Is there a completed descriptor? */ if (pipe->sys.no_queue) read_offset = bam_pipe_get_desc_read_offset(dev->base, pipe_index); else read_offset = pipe->sys.cache_offset; if (read_offset == pipe->sys.acked_offset) { /* No, so clear the iovec to indicate FIFO is empty */ memset(iovec, 0, sizeof(*iovec)); return 0; } /* Fetch next descriptor */ desc = (struct sps_iovec *) (pipe->sys.desc_buf + pipe->sys.acked_offset); *iovec = *desc; #ifdef SPS_BAM_STATISTICS pipe->sys.get_iovecs++; #endif /* SPS_BAM_STATISTICS */ /* Update read/ACK offset */ pipe->sys.acked_offset += sizeof(struct sps_iovec); if (pipe->sys.acked_offset >= pipe->desc_size) pipe->sys.acked_offset = 0; return 0; } /** * Determine whether a BAM pipe descriptor FIFO is empty * */ int sps_bam_pipe_is_empty(struct sps_bam *dev, u32 pipe_index, u32 *empty) { struct sps_pipe *pipe = dev->pipes[pipe_index]; u32 end_offset; u32 acked_offset; /* Is this a satellite connection? */ if ((pipe->state & BAM_STATE_REMOTE)) { SPS_ERR("sps:Is empty on remote: BAM 0x%x pipe %d\n", BAM_ID(dev), pipe_index); return SPS_ERROR; } /* Get offset of last descriptor completed by the pipe */ end_offset = bam_pipe_get_desc_read_offset(dev->base, pipe_index); if ((pipe->state & BAM_STATE_BAM2BAM) == 0) /* System mode */ acked_offset = pipe->sys.acked_offset; else /* BAM-to-BAM */ acked_offset = bam_pipe_get_desc_write_offset(dev->base, pipe_index); /* Determine descriptor FIFO state */ if (end_offset == acked_offset) *empty = true; else *empty = false; return 0; } /** * Get number of free slots in a BAM pipe descriptor FIFO * */ int sps_bam_get_free_count(struct sps_bam *dev, u32 pipe_index, u32 *count) { struct sps_pipe *pipe = dev->pipes[pipe_index]; u32 next_write; u32 free; /* Is this a BAM-to-BAM or satellite connection? */ if ((pipe->state & (BAM_STATE_BAM2BAM | BAM_STATE_REMOTE))) { SPS_ERR( "sps:Free count on BAM-to-BAM or remote: BAM 0x%x pipe %d\n", BAM_ID(dev), pipe_index); *count = 0; return SPS_ERROR; } /* Determine descriptor FIFO state */ next_write = pipe->sys.desc_offset + sizeof(struct sps_iovec); if (next_write >= pipe->desc_size) next_write = 0; if (pipe->sys.acked_offset >= next_write) free = pipe->sys.acked_offset - next_write; else free = pipe->desc_size - next_write + pipe->sys.acked_offset; free /= sizeof(struct sps_iovec); *count = free; return 0; } /** * Set BAM pipe to satellite ownership * */ int sps_bam_set_satellite(struct sps_bam *dev, u32 pipe_index) { struct sps_pipe *pipe = dev->pipes[pipe_index]; /* * Switch to satellite control is only supported on processor * that controls the BAM global config on multi-EE BAMs */ if ((dev->props.manage & SPS_BAM_MGR_MULTI_EE) == 0 || (dev->props.manage & SPS_BAM_MGR_DEVICE_REMOTE)) { SPS_ERR( "sps:Cannot grant satellite control to BAM 0x%x pipe %d\n", BAM_ID(dev), pipe_index); return SPS_ERROR; } /* Is this pipe locally controlled? */ if ((dev->pipe_active_mask & (1UL << pipe_index)) == 0) { SPS_ERR("sps:BAM 0x%x pipe %d not local and active\n", BAM_ID(dev), pipe_index); return SPS_ERROR; } /* Disable local interrupts for this pipe */ if (!pipe->polled) bam_pipe_set_irq(dev->base, pipe_index, BAM_DISABLE, pipe->irq_mask, dev->props.ee); if (BAM_VERSION_MTI_SUPPORT(dev->version)) { /* * Set pipe to MTI interrupt mode. * Must be performed after IRQ disable, * because it is necessary to re-enable the IRQ to enable * MTI generation. * Set both pipe IRQ mask and MTI dest address to zero. */ if ((pipe->state & BAM_STATE_MTI) == 0 || pipe->polled) { bam_pipe_satellite_mti(dev->base, pipe_index, 0, dev->props.ee); pipe->state |= BAM_STATE_MTI; } } /* Indicate satellite control */ list_del(&pipe->list); dev->pipe_active_mask &= ~(1UL << pipe_index); dev->pipe_remote_mask |= pipe->pipe_index_mask; pipe->state |= BAM_STATE_REMOTE; return 0; } /** * Perform BAM pipe timer control * */ int sps_bam_pipe_timer_ctrl(struct sps_bam *dev, u32 pipe_index, struct sps_timer_ctrl *timer_ctrl, struct sps_timer_result *timer_result) { enum bam_pipe_timer_mode mode; int result = 0; /* Is this pipe locally controlled? */ if ((dev->pipe_active_mask & (1UL << pipe_index)) == 0) { SPS_ERR("sps:BAM 0x%x pipe %d not local and active\n", BAM_ID(dev), pipe_index); return SPS_ERROR; } /* Perform the timer operation */ switch (timer_ctrl->op) { case SPS_TIMER_OP_CONFIG: mode = (timer_ctrl->mode == SPS_TIMER_MODE_ONESHOT) ? BAM_PIPE_TIMER_ONESHOT : BAM_PIPE_TIMER_PERIODIC; bam_pipe_timer_config(dev->base, pipe_index, mode, timer_ctrl->timeout_msec * 8); break; case SPS_TIMER_OP_RESET: bam_pipe_timer_reset(dev->base, pipe_index); break; case SPS_TIMER_OP_READ: break; default: result = SPS_ERROR; break; } /* Provide the current timer value */ if (timer_result != NULL) timer_result->current_timer = bam_pipe_timer_get_count(dev->base, pipe_index); return result; } /** * Get the number of unused descriptors in the descriptor FIFO * of a pipe */ int sps_bam_pipe_get_unused_desc_num(struct sps_bam *dev, u32 pipe_index, u32 *desc_num) { u32 sw_offset, peer_offset, fifo_size; u32 desc_size = sizeof(struct sps_iovec); struct sps_pipe *pipe = dev->pipes[pipe_index]; if (pipe == NULL) return SPS_ERROR; fifo_size = pipe->desc_size; sw_offset = bam_pipe_get_desc_read_offset(dev->base, pipe_index); peer_offset = bam_pipe_get_desc_write_offset(dev->base, pipe_index); if (sw_offset <= peer_offset) *desc_num = (peer_offset - sw_offset) / desc_size; else *desc_num = (peer_offset + fifo_size - sw_offset) / desc_size; return 0; }