diff --git a/android/abi_gki_aarch64.xml b/android/abi_gki_aarch64.xml index 72f0674778bc..5765c9ea961e 100644 --- a/android/abi_gki_aarch64.xml +++ b/android/abi_gki_aarch64.xml @@ -53,6 +53,7 @@ + @@ -145,6 +146,7 @@ + @@ -977,6 +979,7 @@ + @@ -2247,6 +2250,7 @@ + @@ -2615,7 +2619,9 @@ + + @@ -2839,6 +2845,7 @@ + @@ -2851,10 +2858,12 @@ + + @@ -2913,6 +2922,7 @@ + @@ -4696,6 +4706,7 @@ + @@ -18363,7 +18374,7 @@ - + @@ -18379,7 +18390,7 @@ - + @@ -29058,7 +29069,14 @@ - + + + + + + + + @@ -29878,7 +29896,7 @@ - + @@ -40682,7 +40700,35 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -41503,6 +41549,9 @@ + + + @@ -44704,6 +44753,7 @@ + @@ -105266,6 +105316,13 @@ + + + + + + + @@ -106068,6 +106125,23 @@ + + + + + + + + + + + + + + + + + @@ -125383,23 +125457,23 @@ - - + + - - - - - + + + + + - - + + - - + + @@ -125407,21 +125481,21 @@ - - - + + + - - + + - - + + - - + + @@ -129275,7 +129349,7 @@ - + @@ -164703,10 +164777,10 @@ - - - - + + + + @@ -167809,6 +167883,11 @@ + + + + + @@ -167946,6 +168025,11 @@ + + + + + @@ -168861,12 +168945,28 @@ + + + + + + + + + + + + + + + + @@ -169353,7 +169453,14 @@ - + + + + + + + + @@ -170716,6 +170823,15 @@ + + + + + + + + + diff --git a/android/abi_gki_aarch64_qcom b/android/abi_gki_aarch64_qcom index 6a6635dbe938..329a7fc9e442 100644 --- a/android/abi_gki_aarch64_qcom +++ b/android/abi_gki_aarch64_qcom @@ -377,6 +377,7 @@ dev_alloc_name dev_coredumpv _dev_crit + __dev_direct_xmit dev_driver_string _dev_emerg _dev_err @@ -393,6 +394,7 @@ __dev_get_by_index dev_get_by_index dev_get_by_name + dev_get_by_name_rcu dev_get_regmap device_add device_add_disk @@ -862,6 +864,7 @@ drm_universal_plane_init drm_vblank_init drm_wait_one_vblank + dst_release dump_stack __dynamic_dev_dbg __dynamic_pr_debug @@ -1170,6 +1173,7 @@ __iowrite32_copy ip_compute_csum ipi_desc_get + ip_route_output_flow iput __ipv6_addr_type ipv6_ext_hdr @@ -1438,7 +1442,10 @@ napi_gro_receive __napi_schedule napi_schedule_prep + neigh_destroy + __neigh_event_send neigh_lookup + neigh_xmit __netdev_alloc_skb netdev_rx_handler_register netdev_rx_handler_unregister @@ -1621,20 +1628,26 @@ param_set_copystring param_set_int pause_cpus + pci_aer_clear_nonfatal_status pci_alloc_irq_vectors_affinity pci_assign_resource pci_bus_type pci_clear_master pci_d3cold_disable pci_device_group + pci_device_is_present pci_dev_present pci_disable_device pci_disable_msi + pci_disable_pcie_error_reporting + pcie_capability_clear_and_set_word pcie_capability_read_word pci_enable_device + pci_enable_pcie_error_reporting pci_find_ext_capability pci_free_irq_vectors pci_get_device + pci_get_domain_bus_and_slot pci_host_probe pci_iomap pci_irq_vector @@ -2366,6 +2379,7 @@ task_may_not_preempt __task_pid_nr_ns __task_rq_lock + thermal_cooling_device_register thermal_cooling_device_unregister thermal_of_cooling_device_register thermal_pressure @@ -2561,6 +2575,7 @@ __tracepoint_android_vh_jiffies_update __tracepoint_android_vh_logbuf __tracepoint_android_vh_printk_hotplug + __tracepoint_android_vh_process_killed __tracepoint_android_vh_psi_event __tracepoint_android_vh_psi_group __tracepoint_android_vh_scheduler_tick diff --git a/arch/arm64/configs/gki_defconfig b/arch/arm64/configs/gki_defconfig index 6743fa469da2..69a9e94cf848 100644 --- a/arch/arm64/configs/gki_defconfig +++ b/arch/arm64/configs/gki_defconfig @@ -228,8 +228,11 @@ CONFIG_NET_SCHED=y CONFIG_NET_SCH_HTB=y CONFIG_NET_SCH_PRIO=y CONFIG_NET_SCH_MULTIQ=y +CONFIG_NET_SCH_SFQ=y CONFIG_NET_SCH_TBF=y CONFIG_NET_SCH_NETEM=y +CONFIG_NET_SCH_CODEL=y +CONFIG_NET_SCH_FQ_CODEL=y CONFIG_NET_SCH_INGRESS=y CONFIG_NET_CLS_BASIC=y CONFIG_NET_CLS_TCINDEX=y @@ -318,10 +321,12 @@ CONFIG_PPP_DEFLATE=y CONFIG_PPP_MPPE=y CONFIG_PPTP=y CONFIG_PPPOL2TP=y +CONFIG_USB_RTL8150=y CONFIG_USB_RTL8152=y CONFIG_USB_USBNET=y # CONFIG_USB_NET_AX8817X is not set # CONFIG_USB_NET_AX88179_178A is not set +CONFIG_USB_NET_CDC_EEM=y # CONFIG_USB_NET_NET1080 is not set # CONFIG_USB_NET_CDC_SUBSET is not set # CONFIG_USB_NET_ZAURUS is not set @@ -476,7 +481,9 @@ CONFIG_USB_CONFIGFS_UEVENT=y CONFIG_USB_CONFIGFS_SERIAL=y CONFIG_USB_CONFIGFS_ACM=y CONFIG_USB_CONFIGFS_NCM=y +CONFIG_USB_CONFIGFS_ECM=y CONFIG_USB_CONFIGFS_RNDIS=y +CONFIG_USB_CONFIGFS_EEM=y CONFIG_USB_CONFIGFS_MASS_STORAGE=y CONFIG_USB_CONFIGFS_F_FS=y CONFIG_USB_CONFIGFS_F_ACC=y diff --git a/arch/x86/configs/gki_defconfig b/arch/x86/configs/gki_defconfig index 3a9ade6c3ce9..273ce86af4c0 100644 --- a/arch/x86/configs/gki_defconfig +++ b/arch/x86/configs/gki_defconfig @@ -205,8 +205,11 @@ CONFIG_NET_SCHED=y CONFIG_NET_SCH_HTB=y CONFIG_NET_SCH_PRIO=y CONFIG_NET_SCH_MULTIQ=y +CONFIG_NET_SCH_SFQ=y CONFIG_NET_SCH_TBF=y CONFIG_NET_SCH_NETEM=y +CONFIG_NET_SCH_CODEL=y +CONFIG_NET_SCH_FQ_CODEL=y CONFIG_NET_SCH_INGRESS=y CONFIG_NET_CLS_BASIC=y CONFIG_NET_CLS_TCINDEX=y @@ -291,10 +294,12 @@ CONFIG_PPP_DEFLATE=y CONFIG_PPP_MPPE=y CONFIG_PPTP=y CONFIG_PPPOL2TP=y +CONFIG_USB_RTL8150=y CONFIG_USB_RTL8152=y CONFIG_USB_USBNET=y # CONFIG_USB_NET_AX8817X is not set # CONFIG_USB_NET_AX88179_178A is not set +CONFIG_USB_NET_CDC_EEM=y # CONFIG_USB_NET_NET1080 is not set # CONFIG_USB_NET_CDC_SUBSET is not set # CONFIG_USB_NET_ZAURUS is not set @@ -429,7 +434,9 @@ CONFIG_USB_CONFIGFS_UEVENT=y CONFIG_USB_CONFIGFS_SERIAL=y CONFIG_USB_CONFIGFS_ACM=y CONFIG_USB_CONFIGFS_NCM=y +CONFIG_USB_CONFIGFS_ECM=y CONFIG_USB_CONFIGFS_RNDIS=y +CONFIG_USB_CONFIGFS_EEM=y CONFIG_USB_CONFIGFS_MASS_STORAGE=y CONFIG_USB_CONFIGFS_F_FS=y CONFIG_USB_CONFIGFS_F_ACC=y diff --git a/drivers/android/vendor_hooks.c b/drivers/android/vendor_hooks.c index 5bba711af63e..8e4bb3950cca 100644 --- a/drivers/android/vendor_hooks.c +++ b/drivers/android/vendor_hooks.c @@ -314,3 +314,4 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_scmi_timeout_sync); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_find_new_ilb); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_force_compatible_pre); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_force_compatible_post); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_set_balance_anon_file_reclaim); diff --git a/drivers/remoteproc/remoteproc_core.c b/drivers/remoteproc/remoteproc_core.c index 0fc970bfb5d9..9107ec4ca0d7 100644 --- a/drivers/remoteproc/remoteproc_core.c +++ b/drivers/remoteproc/remoteproc_core.c @@ -2421,8 +2421,8 @@ void rproc_report_crash(struct rproc *rproc, enum rproc_crash_type type) dev_err(&rproc->dev, "crash detected in %s: type %s\n", rproc->name, rproc_crash_to_string(type)); - /* create a new task to handle the error */ - schedule_work(&rproc->crash_handler); + /* Have a worker handle the error; ensure system is not suspended */ + queue_work(system_freezable_wq, &rproc->crash_handler); } EXPORT_SYMBOL(rproc_report_crash); diff --git a/drivers/usb/gadget/config.c b/drivers/usb/gadget/config.c index 8bb25773b61e..05507606b2b4 100644 --- a/drivers/usb/gadget/config.c +++ b/drivers/usb/gadget/config.c @@ -164,6 +164,14 @@ int usb_assign_descriptors(struct usb_function *f, { struct usb_gadget *g = f->config->cdev->gadget; + /* super-speed-plus descriptor falls back to super-speed one, + * if such a descriptor was provided, thus avoiding a NULL + * pointer dereference if a 5gbps capable gadget is used with + * a 10gbps capable config (device port + cable + host port) + */ + if (!ssp) + ssp = ss; + if (fs) { f->fs_descriptors = usb_copy_descriptors(fs); if (!f->fs_descriptors) diff --git a/drivers/usb/gadget/function/f_ecm.c b/drivers/usb/gadget/function/f_ecm.c index 7f5cf488b2b1..ffe2486fce71 100644 --- a/drivers/usb/gadget/function/f_ecm.c +++ b/drivers/usb/gadget/function/f_ecm.c @@ -791,7 +791,7 @@ ecm_bind(struct usb_configuration *c, struct usb_function *f) fs_ecm_notify_desc.bEndpointAddress; status = usb_assign_descriptors(f, ecm_fs_function, ecm_hs_function, - ecm_ss_function, NULL); + ecm_ss_function, ecm_ss_function); if (status) goto fail; diff --git a/drivers/usb/gadget/function/f_eem.c b/drivers/usb/gadget/function/f_eem.c index cfcc4e81fb77..e6cb38439c41 100644 --- a/drivers/usb/gadget/function/f_eem.c +++ b/drivers/usb/gadget/function/f_eem.c @@ -302,7 +302,7 @@ static int eem_bind(struct usb_configuration *c, struct usb_function *f) eem_ss_out_desc.bEndpointAddress = eem_fs_out_desc.bEndpointAddress; status = usb_assign_descriptors(f, eem_fs_function, eem_hs_function, - eem_ss_function, NULL); + eem_ss_function, eem_ss_function); if (status) goto fail; diff --git a/drivers/usb/gadget/function/f_hid.c b/drivers/usb/gadget/function/f_hid.c index 1125f4715830..e55699308117 100644 --- a/drivers/usb/gadget/function/f_hid.c +++ b/drivers/usb/gadget/function/f_hid.c @@ -802,7 +802,8 @@ static int hidg_bind(struct usb_configuration *c, struct usb_function *f) hidg_fs_out_ep_desc.bEndpointAddress; status = usb_assign_descriptors(f, hidg_fs_descriptors, - hidg_hs_descriptors, hidg_ss_descriptors, NULL); + hidg_hs_descriptors, hidg_ss_descriptors, + hidg_ss_descriptors); if (status) goto fail; diff --git a/drivers/usb/gadget/function/f_loopback.c b/drivers/usb/gadget/function/f_loopback.c index 1803646b3678..90215a81c178 100644 --- a/drivers/usb/gadget/function/f_loopback.c +++ b/drivers/usb/gadget/function/f_loopback.c @@ -207,7 +207,7 @@ static int loopback_bind(struct usb_configuration *c, struct usb_function *f) ss_loop_sink_desc.bEndpointAddress = fs_loop_sink_desc.bEndpointAddress; ret = usb_assign_descriptors(f, fs_loopback_descs, hs_loopback_descs, - ss_loopback_descs, NULL); + ss_loopback_descs, ss_loopback_descs); if (ret) return ret; diff --git a/drivers/usb/gadget/function/f_ncm.c b/drivers/usb/gadget/function/f_ncm.c index 019bea8e09cc..855127249f24 100644 --- a/drivers/usb/gadget/function/f_ncm.c +++ b/drivers/usb/gadget/function/f_ncm.c @@ -583,7 +583,7 @@ static void ncm_do_notify(struct f_ncm *ncm) data[0] = cpu_to_le32(ncm_bitrate(cdev->gadget)); data[1] = data[0]; - DBG(cdev, "notify speed %d\n", ncm_bitrate(cdev->gadget)); + DBG(cdev, "notify speed %u\n", ncm_bitrate(cdev->gadget)); ncm->notify_state = NCM_NOTIFY_CONNECT; break; } @@ -1101,11 +1101,11 @@ static struct sk_buff *ncm_wrap_ntb(struct gether *port, ncm->ndp_dgram_count = 1; /* Note: we skip opts->next_ndp_index */ - } - /* Delay the timer. */ - hrtimer_start(&ncm->task_timer, TX_TIMEOUT_NSECS, - HRTIMER_MODE_REL_SOFT); + /* Start the timer. */ + hrtimer_start(&ncm->task_timer, TX_TIMEOUT_NSECS, + HRTIMER_MODE_REL_SOFT); + } /* Add the datagram position entries */ ntb_ndp = skb_put_zero(ncm->skb_tx_ndp, dgram_idx_len); diff --git a/drivers/usb/gadget/function/f_printer.c b/drivers/usb/gadget/function/f_printer.c index 61ce8e68f7a3..61bbf27b5203 100644 --- a/drivers/usb/gadget/function/f_printer.c +++ b/drivers/usb/gadget/function/f_printer.c @@ -1101,7 +1101,8 @@ static int printer_func_bind(struct usb_configuration *c, ss_ep_out_desc.bEndpointAddress = fs_ep_out_desc.bEndpointAddress; ret = usb_assign_descriptors(f, fs_printer_function, - hs_printer_function, ss_printer_function, NULL); + hs_printer_function, ss_printer_function, + ss_printer_function); if (ret) return ret; diff --git a/drivers/usb/gadget/function/f_rndis.c b/drivers/usb/gadget/function/f_rndis.c index 0739b05a0ef7..ee95e8f5f9d4 100644 --- a/drivers/usb/gadget/function/f_rndis.c +++ b/drivers/usb/gadget/function/f_rndis.c @@ -789,7 +789,7 @@ rndis_bind(struct usb_configuration *c, struct usb_function *f) ss_notify_desc.bEndpointAddress = fs_notify_desc.bEndpointAddress; status = usb_assign_descriptors(f, eth_fs_function, eth_hs_function, - eth_ss_function, NULL); + eth_ss_function, eth_ss_function); if (status) goto fail; diff --git a/drivers/usb/gadget/function/f_serial.c b/drivers/usb/gadget/function/f_serial.c index e62713846350..1ed8ff0ac2d3 100644 --- a/drivers/usb/gadget/function/f_serial.c +++ b/drivers/usb/gadget/function/f_serial.c @@ -233,7 +233,7 @@ static int gser_bind(struct usb_configuration *c, struct usb_function *f) gser_ss_out_desc.bEndpointAddress = gser_fs_out_desc.bEndpointAddress; status = usb_assign_descriptors(f, gser_fs_function, gser_hs_function, - gser_ss_function, NULL); + gser_ss_function, gser_ss_function); if (status) goto fail; dev_dbg(&cdev->gadget->dev, "generic ttyGS%d: %s speed IN/%s OUT/%s\n", diff --git a/drivers/usb/gadget/function/f_sourcesink.c b/drivers/usb/gadget/function/f_sourcesink.c index ed68a4860b7d..282737e4609c 100644 --- a/drivers/usb/gadget/function/f_sourcesink.c +++ b/drivers/usb/gadget/function/f_sourcesink.c @@ -431,7 +431,8 @@ sourcesink_bind(struct usb_configuration *c, struct usb_function *f) ss_iso_sink_desc.bEndpointAddress = fs_iso_sink_desc.bEndpointAddress; ret = usb_assign_descriptors(f, fs_source_sink_descs, - hs_source_sink_descs, ss_source_sink_descs, NULL); + hs_source_sink_descs, ss_source_sink_descs, + ss_source_sink_descs); if (ret) return ret; diff --git a/drivers/usb/gadget/function/f_subset.c b/drivers/usb/gadget/function/f_subset.c index 4d945254905d..51c1cae162d9 100644 --- a/drivers/usb/gadget/function/f_subset.c +++ b/drivers/usb/gadget/function/f_subset.c @@ -358,7 +358,7 @@ geth_bind(struct usb_configuration *c, struct usb_function *f) fs_subset_out_desc.bEndpointAddress; status = usb_assign_descriptors(f, fs_eth_function, hs_eth_function, - ss_eth_function, NULL); + ss_eth_function, ss_eth_function); if (status) goto fail; diff --git a/drivers/usb/gadget/function/f_tcm.c b/drivers/usb/gadget/function/f_tcm.c index 410fa89eae8f..5a2e9ce2bc35 100644 --- a/drivers/usb/gadget/function/f_tcm.c +++ b/drivers/usb/gadget/function/f_tcm.c @@ -2061,7 +2061,8 @@ static int tcm_bind(struct usb_configuration *c, struct usb_function *f) uasp_fs_cmd_desc.bEndpointAddress = uasp_ss_cmd_desc.bEndpointAddress; ret = usb_assign_descriptors(f, uasp_fs_function_desc, - uasp_hs_function_desc, uasp_ss_function_desc, NULL); + uasp_hs_function_desc, uasp_ss_function_desc, + uasp_ss_function_desc); if (ret) goto ep_fail; diff --git a/drivers/usb/typec/tcpm/tcpm.c b/drivers/usb/typec/tcpm/tcpm.c index 59cec4a4840f..871ba0c42c82 100644 --- a/drivers/usb/typec/tcpm/tcpm.c +++ b/drivers/usb/typec/tcpm/tcpm.c @@ -1992,6 +1992,9 @@ static void vdm_run_state_machine(struct tcpm_port *port) tcpm_log(port, "VDM Tx error, retry"); port->vdm_retries++; port->vdm_state = VDM_STATE_READY; + if (PD_VDO_SVDM(vdo_hdr) && PD_VDO_CMDT(vdo_hdr) == CMDT_INIT) + tcpm_ams_finish(port); + } else { tcpm_ams_finish(port); } break; diff --git a/fs/erofs/compress.h b/fs/erofs/compress.h index 3d452443c545..aea129ddda74 100644 --- a/fs/erofs/compress.h +++ b/fs/erofs/compress.h @@ -26,30 +26,58 @@ struct z_erofs_decompress_req { bool inplace_io, partial_decoding; }; +/* some special page->private (unsigned long, see below) */ +#define Z_EROFS_SHORTLIVED_PAGE (-1UL << 2) +#define Z_EROFS_PREALLOCATED_PAGE (-2UL << 2) + /* - * - 0x5A110C8D ('sallocated', Z_EROFS_MAPPING_STAGING) - - * used to mark temporary allocated pages from other - * file/cached pages and NULL mapping pages. + * For all pages in a pcluster, page->private should be one of + * Type Last 2bits page->private + * short-lived page 00 Z_EROFS_SHORTLIVED_PAGE + * preallocated page (tryalloc) 00 Z_EROFS_PREALLOCATED_PAGE + * cached/managed page 00 pointer to z_erofs_pcluster + * online page (file-backed, 01/10/11 sub-index << 2 | count + * some pages can be used for inplace I/O) + * + * page->mapping should be one of + * Type page->mapping + * short-lived page NULL + * preallocated page NULL + * cached/managed page non-NULL or NULL (invalidated/truncated page) + * online page non-NULL + * + * For all managed pages, PG_private should be set with 1 extra refcount, + * which is used for page reclaim / migration. */ -#define Z_EROFS_MAPPING_STAGING ((void *)0x5A110C8D) -/* check if a page is marked as staging */ -static inline bool z_erofs_page_is_staging(struct page *page) +/* + * short-lived pages are pages directly from buddy system with specific + * page->private (no need to set PagePrivate since these are non-LRU / + * non-movable pages and bypass reclaim / migration code). + */ +static inline bool z_erofs_is_shortlived_page(struct page *page) { - return page->mapping == Z_EROFS_MAPPING_STAGING; -} - -static inline bool z_erofs_put_stagingpage(struct list_head *pagepool, - struct page *page) -{ - if (!z_erofs_page_is_staging(page)) + if (page->private != Z_EROFS_SHORTLIVED_PAGE) return false; - /* staging pages should not be used by others at the same time */ - if (page_ref_count(page) > 1) + DBG_BUGON(page->mapping); + return true; +} + +static inline bool z_erofs_put_shortlivedpage(struct list_head *pagepool, + struct page *page) +{ + if (!z_erofs_is_shortlived_page(page)) + return false; + + /* short-lived pages should not be used by others at the same time */ + if (page_ref_count(page) > 1) { put_page(page); - else + } else { + /* follow the pcluster rule above. */ + set_page_private(page, 0); list_add(&page->lru, pagepool); + } return true; } diff --git a/fs/erofs/decompressor.c b/fs/erofs/decompressor.c index cbadbf55c6c2..04624954a119 100644 --- a/fs/erofs/decompressor.c +++ b/fs/erofs/decompressor.c @@ -28,6 +28,17 @@ struct z_erofs_decompressor { char *name; }; +int z_erofs_load_lz4_config(struct super_block *sb, + struct erofs_super_block *dsb) +{ + u16 distance = le16_to_cpu(dsb->lz4_max_distance); + + EROFS_SB(sb)->lz4.max_distance_pages = distance ? + DIV_ROUND_UP(distance, PAGE_SIZE) + 1 : + LZ4_MAX_DISTANCE_PAGES; + return 0; +} + static int z_erofs_lz4_prepare_destpages(struct z_erofs_decompress_req *rq, struct list_head *pagepool) { @@ -36,6 +47,8 @@ static int z_erofs_lz4_prepare_destpages(struct z_erofs_decompress_req *rq, struct page *availables[LZ4_MAX_DISTANCE_PAGES] = { NULL }; unsigned long bounced[DIV_ROUND_UP(LZ4_MAX_DISTANCE_PAGES, BITS_PER_LONG)] = { 0 }; + unsigned int lz4_max_distance_pages = + EROFS_SB(rq->sb)->lz4.max_distance_pages; void *kaddr = NULL; unsigned int i, j, top; @@ -44,14 +57,14 @@ static int z_erofs_lz4_prepare_destpages(struct z_erofs_decompress_req *rq, struct page *const page = rq->out[i]; struct page *victim; - if (j >= LZ4_MAX_DISTANCE_PAGES) + if (j >= lz4_max_distance_pages) j = 0; /* 'valid' bounced can only be tested after a complete round */ if (test_bit(j, bounced)) { - DBG_BUGON(i < LZ4_MAX_DISTANCE_PAGES); - DBG_BUGON(top >= LZ4_MAX_DISTANCE_PAGES); - availables[top++] = rq->out[i - LZ4_MAX_DISTANCE_PAGES]; + DBG_BUGON(i < lz4_max_distance_pages); + DBG_BUGON(top >= lz4_max_distance_pages); + availables[top++] = rq->out[i - lz4_max_distance_pages]; } if (page) { @@ -73,10 +86,9 @@ static int z_erofs_lz4_prepare_destpages(struct z_erofs_decompress_req *rq, victim = availables[--top]; get_page(victim); } else { - victim = erofs_allocpage(pagepool, GFP_KERNEL); - if (!victim) - return -ENOMEM; - victim->mapping = Z_EROFS_MAPPING_STAGING; + victim = erofs_allocpage(pagepool, + GFP_KERNEL | __GFP_NOFAIL); + set_page_private(victim, Z_EROFS_SHORTLIVED_PAGE); } rq->out[i] = victim; } diff --git a/fs/erofs/erofs_fs.h b/fs/erofs/erofs_fs.h index e8d04d808fa6..dc7cc79410df 100644 --- a/fs/erofs/erofs_fs.h +++ b/fs/erofs/erofs_fs.h @@ -39,7 +39,9 @@ struct erofs_super_block { __u8 uuid[16]; /* 128-bit uuid for volume */ __u8 volume_name[16]; /* volume name */ __le32 feature_incompat; - __u8 reserved2[44]; + /* customized lz4 sliding window size instead of 64k by default */ + __le16 lz4_max_distance; + __u8 reserved2[42]; }; /* diff --git a/fs/erofs/internal.h b/fs/erofs/internal.h index 67a7ec945686..4758b3ceeee4 100644 --- a/fs/erofs/internal.h +++ b/fs/erofs/internal.h @@ -50,6 +50,8 @@ struct erofs_fs_context { #ifdef CONFIG_EROFS_FS_ZIP /* current strategy of how to use managed cache */ unsigned char cache_strategy; + /* strategy of sync decompression (false - auto, true - force on) */ + bool readahead_sync_decompress; /* threshold for decompression synchronously */ unsigned int max_sync_decompress_pages; @@ -57,6 +59,12 @@ struct erofs_fs_context { unsigned int mount_opt; }; +/* all filesystem-wide lz4 configurations */ +struct erofs_sb_lz4_info { + /* # of pages needed for EROFS lz4 rolling decompression */ + u16 max_distance_pages; +}; + struct erofs_sb_info { #ifdef CONFIG_EROFS_FS_ZIP /* list for all registered superblocks, mainly for shrinker */ @@ -70,6 +78,8 @@ struct erofs_sb_info { /* pseudo inode to manage cached pages */ struct inode *managed_cache; + + struct erofs_sb_lz4_info lz4; #endif /* CONFIG_EROFS_FS_ZIP */ u32 blocks; u32 meta_blkaddr; @@ -420,6 +430,8 @@ int erofs_try_to_free_all_cached_pages(struct erofs_sb_info *sbi, struct erofs_workgroup *egrp); int erofs_try_to_free_cached_page(struct address_space *mapping, struct page *page); +int z_erofs_load_lz4_config(struct super_block *sb, + struct erofs_super_block *dsb); #else static inline void erofs_shrinker_register(struct super_block *sb) {} static inline void erofs_shrinker_unregister(struct super_block *sb) {} @@ -427,6 +439,15 @@ static inline int erofs_init_shrinker(void) { return 0; } static inline void erofs_exit_shrinker(void) {} static inline int z_erofs_init_zip_subsystem(void) { return 0; } static inline void z_erofs_exit_zip_subsystem(void) {} +static inline int z_erofs_load_lz4_config(struct super_block *sb, + struct erofs_super_block *dsb) +{ + if (dsb->lz4_max_distance) { + erofs_err(sb, "lz4 algorithm isn't enabled"); + return -EINVAL; + } + return 0; +} #endif /* !CONFIG_EROFS_FS_ZIP */ #define EFSCORRUPTED EUCLEAN /* Filesystem is corrupted */ diff --git a/fs/erofs/super.c b/fs/erofs/super.c index d5a6b9b888a5..6ecb5d7b6e0f 100644 --- a/fs/erofs/super.c +++ b/fs/erofs/super.c @@ -187,7 +187,9 @@ static int erofs_read_superblock(struct super_block *sb) ret = -EFSCORRUPTED; goto out; } - ret = 0; + + /* parse on-disk compression configurations */ + ret = z_erofs_load_lz4_config(sb, dsb); out: kunmap(page); put_page(page); @@ -200,6 +202,7 @@ static void erofs_default_options(struct erofs_fs_context *ctx) #ifdef CONFIG_EROFS_FS_ZIP ctx->cache_strategy = EROFS_ZIP_CACHE_READAROUND; ctx->max_sync_decompress_pages = 3; + ctx->readahead_sync_decompress = false; #endif #ifdef CONFIG_EROFS_FS_XATTR set_opt(ctx, XATTR_USER); diff --git a/fs/erofs/zdata.c b/fs/erofs/zdata.c index 86fd3bf62af6..59a969d7ed1f 100644 --- a/fs/erofs/zdata.c +++ b/fs/erofs/zdata.c @@ -20,6 +20,11 @@ enum z_erofs_cache_alloctype { DONTALLOC, /* don't allocate any cached pages */ DELAYEDALLOC, /* delayed allocation (at the time of submitting io) */ + /* + * try to use cached I/O if page allocation succeeds or fallback + * to in-place I/O instead to avoid any direct reclaim. + */ + TRYALLOC, }; /* @@ -154,13 +159,16 @@ static DEFINE_MUTEX(z_pagemap_global_lock); static void preload_compressed_pages(struct z_erofs_collector *clt, struct address_space *mc, - enum z_erofs_cache_alloctype type) + enum z_erofs_cache_alloctype type, + struct list_head *pagepool) { const struct z_erofs_pcluster *pcl = clt->pcl; const unsigned int clusterpages = BIT(pcl->clusterbits); struct page **pages = clt->compressedpages; pgoff_t index = pcl->obj.index + (pages - pcl->compressed_pages); bool standalone = true; + gfp_t gfp = (mapping_gfp_mask(mc) & ~__GFP_DIRECT_RECLAIM) | + __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN; if (clt->mode < COLLECT_PRIMARY_FOLLOWED) return; @@ -168,6 +176,7 @@ static void preload_compressed_pages(struct z_erofs_collector *clt, for (; pages < pcl->compressed_pages + clusterpages; ++pages) { struct page *page; compressed_page_t t; + struct page *newpage = NULL; /* the compressed page was loaded before */ if (READ_ONCE(*pages)) @@ -179,7 +188,15 @@ static void preload_compressed_pages(struct z_erofs_collector *clt, t = tag_compressed_page_justfound(page); } else if (type == DELAYEDALLOC) { t = tagptr_init(compressed_page_t, PAGE_UNALLOCATED); + } else if (type == TRYALLOC) { + newpage = erofs_allocpage(pagepool, gfp); + if (!newpage) + goto dontalloc; + + set_page_private(newpage, Z_EROFS_PREALLOCATED_PAGE); + t = tag_compressed_page_justfound(newpage); } else { /* DONTALLOC */ +dontalloc: if (standalone) clt->compressedpages = pages; standalone = false; @@ -189,8 +206,12 @@ static void preload_compressed_pages(struct z_erofs_collector *clt, if (!cmpxchg_relaxed(pages, NULL, tagptr_cast_ptr(t))) continue; - if (page) + if (page) { put_page(page); + } else if (newpage) { + set_page_private(newpage, 0); + list_add(&newpage->lru, pagepool); + } } if (standalone) /* downgrade to PRIMARY_FOLLOWED_NOINPLACE */ @@ -226,11 +247,8 @@ int erofs_try_to_free_all_cached_pages(struct erofs_sb_info *sbi, /* barrier is implied in the following 'unlock_page' */ WRITE_ONCE(pcl->compressed_pages[i], NULL); - set_page_private(page, 0); - ClearPagePrivate(page); - + detach_page_private(page); unlock_page(page); - put_page(page); } return 0; } @@ -254,10 +272,8 @@ int erofs_try_to_free_cached_page(struct address_space *mapping, } erofs_workgroup_unfreeze(&pcl->obj, 1); - if (ret) { - ClearPagePrivate(page); - put_page(page); - } + if (ret) + detach_page_private(page); } return ret; } @@ -562,7 +578,7 @@ static bool should_alloc_managed_pages(struct z_erofs_decompress_frontend *fe, } static int z_erofs_do_read_page(struct z_erofs_decompress_frontend *fe, - struct page *page) + struct page *page, struct list_head *pagepool) { struct inode *const inode = fe->inode; struct erofs_sb_info *const sbi = EROFS_I_SB(inode); @@ -615,11 +631,12 @@ static int z_erofs_do_read_page(struct z_erofs_decompress_frontend *fe, /* preload all compressed pages (maybe downgrade role if necessary) */ if (should_alloc_managed_pages(fe, sbi->ctx.cache_strategy, map->m_la)) - cache_strategy = DELAYEDALLOC; + cache_strategy = TRYALLOC; else cache_strategy = DONTALLOC; - preload_compressed_pages(clt, MNGD_MAPPING(sbi), cache_strategy); + preload_compressed_pages(clt, MNGD_MAPPING(sbi), + cache_strategy, pagepool); hitted: /* @@ -648,12 +665,12 @@ static int z_erofs_do_read_page(struct z_erofs_decompress_frontend *fe, retry: err = z_erofs_attach_page(clt, page, page_type); - /* should allocate an additional staging page for pagevec */ + /* should allocate an additional short-lived page for pagevec */ if (err == -EAGAIN) { struct page *const newpage = alloc_page(GFP_NOFS | __GFP_NOFAIL); - newpage->mapping = Z_EROFS_MAPPING_STAGING; + set_page_private(newpage, Z_EROFS_SHORTLIVED_PAGE); err = z_erofs_attach_page(clt, newpage, Z_EROFS_PAGE_TYPE_EXCLUSIVE); if (!err) @@ -692,9 +709,12 @@ static int z_erofs_do_read_page(struct z_erofs_decompress_frontend *fe, goto out; } +static void z_erofs_decompressqueue_work(struct work_struct *work); static void z_erofs_decompress_kickoff(struct z_erofs_decompressqueue *io, bool sync, int bios) { + struct erofs_sb_info *const sbi = EROFS_SB(io->sb); + /* wake up the caller thread for sync decompression */ if (sync) { unsigned long flags; @@ -706,8 +726,20 @@ static void z_erofs_decompress_kickoff(struct z_erofs_decompressqueue *io, return; } - if (!atomic_add_return(bios, &io->pending_bios)) + if (atomic_add_return(bios, &io->pending_bios)) + return; + /* Use workqueue and sync decompression for atomic contexts only */ + if (in_atomic() || irqs_disabled()) { queue_work(z_erofs_workqueue, &io->u.work); + sbi->ctx.readahead_sync_decompress = true; + return; + } + z_erofs_decompressqueue_work(&io->u.work); +} + +static bool z_erofs_page_is_invalidated(struct page *page) +{ + return !page->mapping && !z_erofs_is_shortlived_page(page); } static void z_erofs_decompressqueue_endio(struct bio *bio) @@ -722,7 +754,7 @@ static void z_erofs_decompressqueue_endio(struct bio *bio) struct page *page = bvec->bv_page; DBG_BUGON(PageUptodate(page)); - DBG_BUGON(!page->mapping); + DBG_BUGON(z_erofs_page_is_invalidated(page)); if (err) SetPageError(page); @@ -795,9 +827,9 @@ static int z_erofs_decompress_pcluster(struct super_block *sb, /* all pages in pagevec ought to be valid */ DBG_BUGON(!page); - DBG_BUGON(!page->mapping); + DBG_BUGON(z_erofs_page_is_invalidated(page)); - if (z_erofs_put_stagingpage(pagepool, page)) + if (z_erofs_put_shortlivedpage(pagepool, page)) continue; if (page_type == Z_EROFS_VLE_PAGE_TYPE_HEAD) @@ -831,9 +863,9 @@ static int z_erofs_decompress_pcluster(struct super_block *sb, /* all compressed pages ought to be valid */ DBG_BUGON(!page); - DBG_BUGON(!page->mapping); + DBG_BUGON(z_erofs_page_is_invalidated(page)); - if (!z_erofs_page_is_staging(page)) { + if (!z_erofs_is_shortlived_page(page)) { if (erofs_page_is_managed(sbi, page)) { if (!PageUptodate(page)) err = -EIO; @@ -858,7 +890,7 @@ static int z_erofs_decompress_pcluster(struct super_block *sb, overlapped = true; } - /* PG_error needs checking for inplaced and staging pages */ + /* PG_error needs checking for all non-managed pages */ if (PageError(page)) { DBG_BUGON(PageUptodate(page)); err = -EIO; @@ -897,8 +929,8 @@ static int z_erofs_decompress_pcluster(struct super_block *sb, if (erofs_page_is_managed(sbi, page)) continue; - /* recycle all individual staging pages */ - (void)z_erofs_put_stagingpage(pagepool, page); + /* recycle all individual short-lived pages */ + (void)z_erofs_put_shortlivedpage(pagepool, page); WRITE_ONCE(compressed_pages[i], NULL); } @@ -908,10 +940,10 @@ static int z_erofs_decompress_pcluster(struct super_block *sb, if (!page) continue; - DBG_BUGON(!page->mapping); + DBG_BUGON(z_erofs_page_is_invalidated(page)); - /* recycle all individual staging pages */ - if (z_erofs_put_stagingpage(pagepool, page)) + /* recycle all individual short-lived pages */ + if (z_erofs_put_shortlivedpage(pagepool, page)) continue; if (err < 0) @@ -1008,16 +1040,30 @@ static struct page *pickup_page_for_submission(struct z_erofs_pcluster *pcl, justfound = tagptr_unfold_tags(t); page = tagptr_unfold_ptr(t); + /* + * preallocated cached pages, which is used to avoid direct reclaim + * otherwise, it will go inplace I/O path instead. + */ + if (page->private == Z_EROFS_PREALLOCATED_PAGE) { + WRITE_ONCE(pcl->compressed_pages[nr], page); + set_page_private(page, 0); + tocache = true; + goto out_tocache; + } mapping = READ_ONCE(page->mapping); /* - * unmanaged (file) pages are all locked solidly, + * file-backed online pages in plcuster are all locked steady, * therefore it is impossible for `mapping' to be NULL. */ if (mapping && mapping != mc) /* ought to be unmanaged pages */ goto out; + /* directly return for shortlived page as well */ + if (z_erofs_is_shortlived_page(page)) + goto out; + lock_page(page); /* only true if page reclaim goes wrong, should never happen */ @@ -1061,28 +1107,21 @@ static struct page *pickup_page_for_submission(struct z_erofs_pcluster *pcl, put_page(page); out_allocpage: page = erofs_allocpage(pagepool, gfp | __GFP_NOFAIL); - if (!tocache || add_to_page_cache_lru(page, mc, index + nr, gfp)) { - /* non-LRU / non-movable temporary page is needed */ - page->mapping = Z_EROFS_MAPPING_STAGING; - tocache = false; - } - if (oldpage != cmpxchg(&pcl->compressed_pages[nr], oldpage, page)) { - if (tocache) { - /* since it added to managed cache successfully */ - unlock_page(page); - put_page(page); - } else { - list_add(&page->lru, pagepool); - } + list_add(&page->lru, pagepool); cond_resched(); goto repeat; } - - if (tocache) { - set_page_private(page, (unsigned long)pcl); - SetPagePrivate(page); +out_tocache: + if (!tocache || add_to_page_cache_lru(page, mc, index + nr, gfp)) { + /* turn into temporary page if fails (1 ref) */ + set_page_private(page, Z_EROFS_SHORTLIVED_PAGE); + goto out; } + attach_page_private(page, pcl); + /* drop a refcount added by allocpage (then we have 2 refs here) */ + put_page(page); + out: /* the only exit (for tracing and debugging) */ return page; } @@ -1284,7 +1323,7 @@ static int z_erofs_readpage(struct file *file, struct page *page) f.headoffset = (erofs_off_t)page->index << PAGE_SHIFT; - err = z_erofs_do_read_page(&f, page); + err = z_erofs_do_read_page(&f, page, &pagepool); (void)z_erofs_collector_end(&f.clt); /* if some compressed cluster ready, need submit them anyway */ @@ -1307,7 +1346,8 @@ static void z_erofs_readahead(struct readahead_control *rac) struct erofs_sb_info *const sbi = EROFS_I_SB(inode); unsigned int nr_pages = readahead_count(rac); - bool sync = (nr_pages <= sbi->ctx.max_sync_decompress_pages); + bool sync = (sbi->ctx.readahead_sync_decompress && + nr_pages <= sbi->ctx.max_sync_decompress_pages); struct z_erofs_decompress_frontend f = DECOMPRESS_FRONTEND_INIT(inode); struct page *page, *head = NULL; LIST_HEAD(pagepool); @@ -1338,7 +1378,7 @@ static void z_erofs_readahead(struct readahead_control *rac) /* traversal in reverse order */ head = (void *)page_private(page); - err = z_erofs_do_read_page(&f, page); + err = z_erofs_do_read_page(&f, page, &pagepool); if (err) erofs_err(inode->i_sb, "readahead error at page %lu @ nid %llu", diff --git a/fs/erofs/zdata.h b/fs/erofs/zdata.h index 68c9b29fc0ca..b503b353d4ab 100644 --- a/fs/erofs/zdata.h +++ b/fs/erofs/zdata.h @@ -173,6 +173,7 @@ static inline void z_erofs_onlinepage_endio(struct page *page) v = atomic_dec_return(u.o); if (!(v & Z_EROFS_ONLINEPAGE_COUNT_MASK)) { + set_page_private(page, 0); ClearPagePrivate(page); if (!PageError(page)) SetPageUptodate(page); diff --git a/include/trace/hooks/vmscan.h b/include/trace/hooks/vmscan.h index a2118ff6901d..b6b77d9895cd 100644 --- a/include/trace/hooks/vmscan.h +++ b/include/trace/hooks/vmscan.h @@ -25,6 +25,9 @@ DECLARE_HOOK(android_vh_tune_inactive_ratio, DECLARE_HOOK(android_vh_do_shrink_slab, TP_PROTO(struct shrinker *shrinker, struct shrink_control *shrinkctl, int priority), TP_ARGS(shrinker, shrinkctl, priority)); +DECLARE_RESTRICTED_HOOK(android_rvh_set_balance_anon_file_reclaim, + TP_PROTO(bool *balance_anon_file_reclaim), + TP_ARGS(balance_anon_file_reclaim), 1); #endif /* _TRACE_HOOK_VMSCAN_H */ /* This part must be outside protection */ #include diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c index ae487015d1c5..f1b193096ffa 100644 --- a/mm/memory_hotplug.c +++ b/mm/memory_hotplug.c @@ -1834,22 +1834,6 @@ int remove_memory(int nid, u64 start, u64 size) } EXPORT_SYMBOL_GPL(remove_memory); -static bool __check_sections_offline(unsigned long start_pfn, - unsigned long nr_pages) -{ - const unsigned long end_pfn = start_pfn + nr_pages; - unsigned long pfn, sec_nr; - - for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) { - sec_nr = pfn_to_section_nr(pfn); - - if (!valid_section_nr(sec_nr) || online_section_nr(sec_nr)) - return false; - } - - return true; -} - int remove_memory_subsection(int nid, u64 start, u64 size) { if (size == memory_block_size_bytes()) @@ -1863,15 +1847,6 @@ int remove_memory_subsection(int nid, u64 start, u64 size) } mem_hotplug_begin(); - - /* we cannot remove subsections that are invalid or online */ - if(!__check_sections_offline(PHYS_PFN(start), size >> PAGE_SHIFT)) { - pr_err("%s: [%llx, %llx) sections are not offlined\n", - __func__, start, start + size); - mem_hotplug_done(); - return -EBUSY; - } - arch_remove_memory(nid, start, size, NULL); if (IS_ENABLED(CONFIG_ARCH_KEEP_MEMBLOCK)) diff --git a/mm/vmscan.c b/mm/vmscan.c index 0fef8daa52f6..d23a602e7d15 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -2272,6 +2272,7 @@ static void get_scan_count(struct lruvec *lruvec, struct scan_control *sc, enum scan_balance scan_balance; unsigned long ap, fp; enum lru_list lru; + bool balance_anon_file_reclaim = false; /* If we have no swap space, do not bother scanning anon pages. */ if (!sc->may_swap || mem_cgroup_get_nr_swap_pages(memcg) <= 0) { @@ -2310,11 +2311,15 @@ static void get_scan_count(struct lruvec *lruvec, struct scan_control *sc, goto out; } + trace_android_rvh_set_balance_anon_file_reclaim(&balance_anon_file_reclaim); + /* * If there is enough inactive page cache, we do not reclaim - * anything from the anonymous working right now. + * anything from the anonymous working right now. But when balancing + * anon and page cache files for reclaim, allow swapping of anon pages + * even if there are a number of inactive file cache pages. */ - if (sc->cache_trim_mode) { + if (!balance_anon_file_reclaim && sc->cache_trim_mode) { scan_balance = SCAN_FILE; goto out; }